Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.2797109 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.022815928 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.022412743 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1189 1190 1191 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.265021045Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:51:41.32720337Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.328362564Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.36405339Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.365633425Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.365689289Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.365707397Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.365725355Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193} {"msg":"Crucible 7a15cbd0-79a8-472a-b560-6d87cfc95cb8 has session id: 75d43d36-fe04-4a60-a1b5-a946e13564e6","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.366582715Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Upstairs opts: Upstairs UUID: 7a15cbd0-79a8-472a-b560-6d87cfc95cb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.366668794Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Crucible stats registered with UUID: 7a15cbd0-79a8-472a-b560-6d87cfc95cb8","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.366700101Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.403897001Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.403975431Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193} {"msg":"7a15cbd0-79a8-472a-b560-6d87cfc95cb8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.404000178Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.404024806Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"0","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.404111425Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"0","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.404135033Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"1","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.40415739Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"1","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.404177568Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"2","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.404230432Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"2","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.404295744Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.404317801Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.404338969Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:65270)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.404624336Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:47240)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.404657992Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:53461)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.404680489Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.406657668Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"0","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.406699543Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"1","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.406724271Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"2","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.40801481Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"1","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.408054595Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"0","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.408079062Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"2","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.408840504Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.408945181Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409156197Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409179554Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409195512Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409263054Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409283122Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.40929773Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409311459Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409328737Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409343095Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409356504Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409369622Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.40938259Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409493477Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"downstairs","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409512535Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409524734Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"0","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409538162Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"1","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409551471Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"client":"2","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409564859Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409577248Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"7a15cbd0-79a8-472a-b560-6d87cfc95cb8 is now active with session: 75d43d36-fe04-4a60-a1b5-a946e13564e6","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409590096Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409601885Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:41.409614293Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:51:45.506145174Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.506220625Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:51:45.506282248Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:51:45.506314794Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.506346161Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"75d43d36-fe04-4a60-a1b5-a946e13564e6"} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.544128492Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:51:45.590214485Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.590737154Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.623224113Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.623910493Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.623944839Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.623964406Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.623979545Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195} {"msg":"Crucible 9643b891-4c86-4020-bd05-ba77293355ea has session id: 5614b2b7-e77b-46da-9cf7-f774dd853d87","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.624430332Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Upstairs opts: Upstairs UUID: 9643b891-4c86-4020-bd05-ba77293355ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.624474117Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Crucible stats registered with UUID: 9643b891-4c86-4020-bd05-ba77293355ea","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.624500724Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655278063Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655347624Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195} {"msg":"9643b891-4c86-4020-bd05-ba77293355ea active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655367102Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.65538524Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"0","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655414047Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"0","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655432814Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"1","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655450703Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"1","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655468321Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"2","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655485858Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"2","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655502137Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655524054Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.6555553Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"ds_connection connected from Ok(127.0.0.1:35037)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655849676Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"ds_connection connected from Ok(127.0.0.1:39538)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655941115Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"ds_connection connected from Ok(127.0.0.1:56606)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.655977091Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.656558473Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"1","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.656659761Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"2","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.656693867Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"0","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.657209867Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"2","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.65735856Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"1","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.657388926Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"0","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.657839384Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.657932523Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658068407Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658092104Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658107453Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658121501Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658135079Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658148978Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658162856Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658176494Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658190663Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658203861Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.65821703Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658236587Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658249676Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"":"downstairs","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658263254Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658275503Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"0","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658289301Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"1","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.65830455Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"client":"2","":"downstairs","session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658318058Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658330276Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"9643b891-4c86-4020-bd05-ba77293355ea is now active with session: 5614b2b7-e77b-46da-9cf7-f774dd853d87","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658343195Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658355364Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195,"session_id":"5614b2b7-e77b-46da-9cf7-f774dd853d87"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:45.658367662Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 68 len 4096 data: 2 02/30 Write block 189 len 4096 data: 2 03/30 Read block 585 len 16384 04/30 Write block 289 len 20480 data: 2 2 2 2 2 05/30 Write block 114 len 12288 data: 2 2 2 06/30 Write block 50 len 24576 data: 2 2 2 2 2 2 07/30 Write block 7 len 4096 data: 2 08/30 Read block 386 len 8192 09/30 Read block 165 len 20480 10/30 Read block 495 len 16384 11/30 Write block 569 len 8192 data: 2 2 12/30 Write block 64 len 8192 data: 2 2 13/30 Write block 57 len 16384 data: 2 2 2 2 14/30 Read block 511 len 16384 15/30 Read block 542 len 32768 16/30 Read block 492 len 28672 17/30 Write block 239 len 16384 data: 2 2 2 2 18/30 Write block 134 len 36864 data: 2 2 2 2 2 2 2 2 2 19/30 Write block 389 len 36864 data: 2 2 2 2 2 2 2 2 2 20/30 Write block 499 len 40960 data: 2 2 2 2 2 2 2 2 2 2 21/30 Write block 50 len 28672 data: 3 3 3 3 3 3 2 22/30 Read block 178 len 4096 23/30 Write block 567 len 8192 data: 2 2 24/30 Read block 396 len 24576 25/30 Read block 119 len 4096 26/30 Read block 8 len 16384 27/30 Write block 530 len 20480 data: 2 2 2 2 2 28/30 Write block 556 len 12288 data: 2 2 2 29/30 Read block 369 len 16384 30/30 Read block 113 len 28672 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:51:45.675788899Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1195} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T T 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 T T T 3 060-079 1 1 1 1 1 1 T T T 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 T T T 6 120-139 1 1 1 1 1 1 T T T 7 140-159 1 1 1 1 1 1 T T T 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 T T T 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 T T T 12 240-259 1 1 1 1 1 1 T T T 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 T T T 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 T T T 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 T T T 25 500-519 1 1 1 1 1 1 T T T 26 520-539 1 1 1 1 1 1 T T T 27 540-559 1 1 1 1 1 1 T T T 28 560-579 1 1 1 1 1 1 T T T 29 580-599 1 1 1 1 1 1 F F F Max gen: 1, Max flush: 1 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 2 2 2 2 2 2 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 2 2 2 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 2 2 2 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 2 2 2 2 2 2 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 2 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.043002395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.086393182Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.08718366Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.087229254Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.087247122Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.08726459Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203} {"msg":"Crucible 2feea48d-ff85-4a04-8c2a-3320c9396c19 has session id: fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.087767631Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Upstairs opts: Upstairs UUID: 2feea48d-ff85-4a04-8c2a-3320c9396c19, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.087817515Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Crucible stats registered with UUID: 2feea48d-ff85-4a04-8c2a-3320c9396c19","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.087838713Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.117753102Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.117804967Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203} {"msg":"2feea48d-ff85-4a04-8c2a-3320c9396c19 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.117842412Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.117873268Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"0","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.117920823Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"0","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.117958129Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"1","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.117994214Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"1","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.118026271Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"2","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.118061857Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"2","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.118092953Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.1181168Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.118141837Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44259)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.118167884Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:65220)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.118193901Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63194)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.118222718Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.119084427Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"2","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.119112274Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"0","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.119131502Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"1","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.119682288Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"2","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.119759468Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"1","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.119869855Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"0","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120249351Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120330152Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"[0]R flush_numbers[0..12]: [2, 1, 2, 2, 1, 2, 2, 2, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120463446Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"[0]R generation[0..12]: [2, 1, 2, 2, 1, 2, 2, 2, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120483584Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120498512Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"[1]R flush_numbers[0..12]: [2, 1, 2, 2, 1, 2, 2, 2, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.12051278Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"[1]R generation[0..12]: [2, 1, 2, 2, 1, 2, 2, 2, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120526729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120540977Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"[2]R flush_numbers[0..12]: [2, 1, 2, 2, 1, 2, 2, 2, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120570704Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"[2]R generation[0..12]: [2, 1, 2, 2, 1, 2, 2, 2, 1, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120584992Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.12060254Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Max found gen is 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120615928Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Generation requested: 4 >= found:3","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:51:50.120629397Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Next flush: 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120646305Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120660193Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"":"downstairs","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120673612Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.1206856Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"0","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120699109Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"1","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120712257Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"client":"2","":"downstairs","session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120725426Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120737604Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"2feea48d-ff85-4a04-8c2a-3320c9396c19 is now active with session: fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120750343Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.120762391Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203,"session_id":"fa3fe97c-ded0-4128-b5f0-7a50f7d7ff3c"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:50.12077453Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 147 len 28672 data: 2 2 2 2 2 2 2 02/30 Write block 271 len 28672 data: 2 2 2 2 2 2 2 03/30 Write block 432 len 28672 data: 2 2 2 2 2 2 2 04/30 Read block 455 len 24576 05/30 Read block 80 len 16384 06/30 Write block 68 len 32768 data: 3 2 2 2 2 2 2 2 07/30 Write block 43 len 16384 data: 2 2 2 2 08/30 Write block 292 len 28672 data: 3 3 2 2 2 2 2 09/30 Read block 211 len 4096 10/30 Read block 231 len 4096 11/30 Read block 533 len 28672 12/30 Write block 410 len 4096 data: 2 13/30 Flush 14/30 Write block 413 len 32768 data: 2 2 2 2 2 2 2 2 15/30 Write block 351 len 8192 data: 2 2 16/30 Write block 478 len 8192 data: 2 2 17/30 Read block 160 len 40960 18/30 Write block 266 len 28672 data: 2 2 2 2 2 3 3 19/30 Read block 373 len 20480 20/30 Read block 250 len 20480 21/30 Read block 145 len 8192 22/30 Read block 404 len 8192 23/30 Flush 24/30 Flush 25/30 Write block 86 len 40960 data: 2 2 2 2 2 2 2 2 2 2 26/30 Read block 505 len 24576 27/30 Write block 516 len 8192 data: 2 2 28/30 Read block 412 len 40960 29/30 Write block 188 len 4096 data: 2 30/30 Read block 446 len 4096 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 6 6 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:51:50.139671485Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1203} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 4 4 4 3 3 3 F F F 3 060-079 4 4 4 3 3 3 F F F 4 080-099 1 1 1 1 1 1 T T T 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 4 4 4 3 3 3 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 2 2 2 2 2 2 T T T 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 2 2 2 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 4 4 4 4 4 4 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 4 4 4 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 4 4 4 4 4 4 F F F 21 420-439 4 4 4 4 4 4 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 4 4 4 4 4 4 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 2 2 2 2 2 2 T T T 26 520-539 2 2 2 2 2 2 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 4 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 4 4 4 3 3 3 F F F 3 060-079 4 4 4 3 3 3 F F F 4 080-099 5 5 5 5 5 5 F F F 5 100-119 2 2 2 2 2 2 F F F 6 120-139 2 2 2 2 2 2 F F F 7 140-159 4 4 4 3 3 3 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 5 5 5 5 5 5 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 2 2 2 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 4 4 4 4 4 4 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 4 4 4 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 4 4 4 4 4 4 F F F 21 420-439 4 4 4 4 4 4 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 4 4 4 4 4 4 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 5 5 5 5 5 5 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 5 Begin loop 3 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.405638315Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:51:54.448829295Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.449361143Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.481590722Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.482364512Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.482401908Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.482415976Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.482431075Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211} {"msg":"Crucible 9e32446c-2dfa-482a-be99-130ce7b7a681 has session id: 6fdd44e0-2311-448d-9bba-f5ea16e2318d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.482991899Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Upstairs opts: Upstairs UUID: 9e32446c-2dfa-482a-be99-130ce7b7a681, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.483042773Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Crucible stats registered with UUID: 9e32446c-2dfa-482a-be99-130ce7b7a681","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.48306493Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.513744101Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.513786486Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211} {"msg":"9e32446c-2dfa-482a-be99-130ce7b7a681 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.513868246Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.513891784Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"0","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.51392308Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"0","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.513955256Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"1","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.513991742Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"1","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.514027028Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"2","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.514062253Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"2","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.51409485Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"io task","client":"2","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.514136245Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"io task","client":"0","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.514174591Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"io task","client":"1","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34510)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.514216515Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"io task","client":"1","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41830)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.514252801Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"io task","client":"0","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41741)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.514468436Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"io task","client":"2","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.515091244Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"1","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.515125169Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"2","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.515213589Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"0","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.515828467Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"1","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.51597399Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"2","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516036953Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"0","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.51631623Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516503899Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"[0]R flush_numbers[0..12]: [2, 1, 3, 3, 5, 2, 2, 3, 1, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516631264Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"[0]R generation[0..12]: [2, 1, 4, 4, 5, 2, 2, 4, 1, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516652411Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516667849Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"[1]R flush_numbers[0..12]: [2, 1, 3, 3, 5, 2, 2, 3, 1, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516681838Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"[1]R generation[0..12]: [2, 1, 4, 4, 5, 2, 2, 4, 1, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516699606Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516714284Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"[2]R flush_numbers[0..12]: [2, 1, 3, 3, 5, 2, 2, 3, 1, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516728033Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"[2]R generation[0..12]: [2, 1, 4, 4, 5, 2, 2, 4, 1, 5, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516741581Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516755529Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516768658Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516781836Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516794775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516814512Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"":"downstairs","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516827851Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.51683984Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"0","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516853378Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"1","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516866976Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"client":"2","":"downstairs","session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516881045Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516893533Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"9e32446c-2dfa-482a-be99-130ce7b7a681 is now active with session: 6fdd44e0-2311-448d-9bba-f5ea16e2318d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516908741Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.51692094Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211,"session_id":"6fdd44e0-2311-448d-9bba-f5ea16e2318d"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:54.516933138Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Read block 216 len 36864 03/30 Write block 16 len 20480 data: 2 2 2 2 2 04/30 Write block 499 len 20480 data: 3 3 3 3 3 05/30 Write block 577 len 4096 data: 2 06/30 Write block 555 len 28672 data: 2 3 3 3 2 2 2 07/30 Read block 134 len 16384 08/30 Read block 430 len 20480 09/30 Read block 130 len 20480 10/30 Write block 514 len 24576 data: 2 2 3 3 2 2 11/30 Write block 183 len 16384 data: 2 2 2 2 12/30 Read block 437 len 40960 13/30 Write block 505 len 32768 data: 3 3 3 3 2 2 2 2 14/30 Read block 269 len 8192 15/30 Flush 16/30 Write block 6 len 36864 data: 2 3 2 2 2 2 2 2 2 17/30 Write block 104 len 20480 data: 2 2 2 2 2 18/30 Write block 301 len 32768 data: 2 2 2 2 2 2 2 2 19/30 Write block 121 len 32768 data: 2 2 2 2 2 2 2 2 20/30 Write block 548 len 32768 data: 2 2 2 2 2 2 2 3 21/30 Flush 22/30 Write block 381 len 32768 data: 2 2 2 2 2 2 2 2 23/30 Write block 264 len 28672 data: 2 2 3 3 3 3 3 24/30 Write block 140 len 24576 data: 3 3 3 2 2 2 25/30 Write block 16 len 20480 data: 3 3 3 3 3 26/30 Read block 244 len 20480 27/30 Write block 26 len 32768 data: 2 2 2 2 2 2 2 2 28/30 Write block 238 len 16384 data: 2 3 3 3 29/30 Read block 52 len 32768 30/30 Read block 338 len 32768 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:51:54.536823838Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1211} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 7 7 7 T T T 1 020-039 6 6 6 6 6 6 T T T 2 040-059 4 4 4 3 3 3 F F F 3 060-079 4 4 4 3 3 3 F F F 4 080-099 5 5 5 5 5 5 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 4 4 4 3 3 3 T T T 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 6 6 6 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 2 2 2 T T T 12 240-259 2 2 2 2 2 2 T T T 13 260-279 4 4 4 4 4 4 T T T 14 280-299 4 4 4 3 3 3 F F F 15 300-319 6 6 6 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 6 6 6 6 6 6 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 2 2 2 T T T 20 400-419 4 4 4 4 4 4 F F F 21 420-439 4 4 4 4 4 4 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 4 4 4 4 4 4 F F F 24 480-499 6 6 6 6 6 6 F F F 25 500-519 6 6 6 6 6 6 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 6 6 6 7 7 7 F F F 28 560-579 6 6 6 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 6, Max flush: 7 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 8 8 8 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 4 4 4 3 3 3 F F F 3 060-079 4 4 4 3 3 3 F F F 4 080-099 5 5 5 5 5 5 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 6 6 6 7 7 7 F F F 7 140-159 7 7 7 8 8 8 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 6 6 6 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 7 7 7 8 8 8 F F F 12 240-259 7 7 7 8 8 8 F F F 13 260-279 7 7 7 8 8 8 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 6 6 6 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 6 6 6 6 6 6 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 4 4 4 4 4 4 F F F 21 420-439 4 4 4 4 4 4 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 4 4 4 4 4 4 F F F 24 480-499 6 6 6 6 6 6 F F F 25 500-519 6 6 6 6 6 6 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 6 6 6 7 7 7 F F F 28 560-579 6 6 6 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 7, Max flush: 8 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.855667186Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:51:58.898608835Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.899028957Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.930155465Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.930955521Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.931002666Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.931036912Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.93105484Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219} {"msg":"Crucible 70dcd631-5067-4508-9524-5ab48cc274e1 has session id: 73faf417-aa91-4591-919c-2bc656c66aec","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.931507007Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Upstairs opts: Upstairs UUID: 70dcd631-5067-4508-9524-5ab48cc274e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.931554891Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Crucible stats registered with UUID: 70dcd631-5067-4508-9524-5ab48cc274e1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.931581628Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.961982841Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.962038754Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219} {"msg":"70dcd631-5067-4508-9524-5ab48cc274e1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.962054642Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.96207103Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"0","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.962101587Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"0","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.962119295Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"1","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.962137223Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"1","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.96215991Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"2","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.962178128Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"2","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.962194646Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.962211924Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.962229032Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"ds_connection connected from Ok(127.0.0.1:44623)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.962382584Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"ds_connection connected from Ok(127.0.0.1:49972)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.962640004Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"ds_connection connected from Ok(127.0.0.1:51919)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.962686538Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.963208008Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"1","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.963251983Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"2","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.963463818Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"0","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.96396287Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"2","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.963992936Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"0","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964027862Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"1","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964566409Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964674686Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"[0]R flush_numbers[0..12]: [8, 8, 3, 3, 5, 7, 7, 8, 1, 6, 1, 8]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964742409Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"[0]R generation[0..12]: [7, 7, 4, 4, 5, 6, 6, 7, 1, 6, 1, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964770875Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964786733Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"[1]R flush_numbers[0..12]: [8, 8, 3, 3, 5, 7, 7, 8, 1, 6, 1, 8]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964800952Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"[1]R generation[0..12]: [7, 7, 4, 4, 5, 6, 6, 7, 1, 6, 1, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.96481469Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964830668Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"[2]R flush_numbers[0..12]: [8, 8, 3, 3, 5, 7, 7, 8, 1, 6, 1, 8]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964844687Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"[2]R generation[0..12]: [7, 7, 4, 4, 5, 6, 6, 7, 1, 6, 1, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964858395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964872423Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964885622Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.96489932Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964912589Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964928637Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"":"downstairs","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964942165Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964954394Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4"Wait for a query_work_queue command to finish before sending IO ,"pid":1219,"client":"0","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964980671Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"1","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.964993979Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"client":"2","":"downstairs","session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.965007508Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.965019926Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"70dcd631-5067-4508-9524-5ab48cc274e1 is now active with session: 73faf417-aa91-4591-919c-2bc656c66aec","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.965032975Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.965045003Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219,"session_id":"73faf417-aa91-4591-919c-2bc656c66aec"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:51:58.965057422Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 317 len 28672 data: 2 2 2 2 2 2 2 02/30 Write block 580 len 32768 data: 2 2 2 2 2 2 2 2 03/30 Write block 217 len 8192 data: 2 2 04/30 Read block 13 len 28672 05/30 Write block 546 len 32768 data: 2 2 3 3 3 3 3 3 06/30 Read block 584 len 8192 07/30 Read block 470 len 4096 08/30 Write block 469 len 40960 data: 2 2 2 2 2 2 2 2 2 3 09/30 Flush 10/30 Write block 583 len 4096 data: 3 11/30 Read block 398 len 20480 12/30 Write block 436 len 32768 data: 3 3 3 2 2 2 2 2 13/30 Read block 485 len 4096 14/30 Write block 230 len 4096 data: 2 15/30 Flush 16/30 Write block 126 len 16384 data: 3 3 3 2 17/30 Write block 500 len 4096 data: 4 18/30 Write block 133 len 16384 data: 2 3 3 3 19/30 Read block 249 len 24576 20/30 Write block 527 len 24576 data: 2 2 2 3 3 3 21/30 Flush 22/30 Write block 587 len 12288 data: 3 2 2 23/30 Write block 532 len 12288 data: 4 3 3 24/30 Write block 460 len 20480 data: 2 2 2 2 2 25/30 Read block 22 len 40960 26/30 Read block 417 len 40960 27/30 Read block 498 len 8192 28/30 Write block 43 len 40960 data: 3 3 3 3 2 2 2 4 4 4 29/30 Read block 296 len 12288 30/30 Read block 91 len 16384 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 3 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:51:58.984469737Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1219} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 8 8 8 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 4 4 4 3 3 3 T T T 3 060-079 4 4 4 3 3 3 F F F 4 080-099 5 5 5 5 5 5 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 8 8 8 11 11 11 F F F 7 140-159 7 7 7 8 8 8 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 6 6 6 F F F 10 200-219 8 8 8 9 9 9 F F F 11 220-239 8 8 8 10 10 10 F F F 12 240-259 7 7 7 8 8 8 F F F 13 260-279 7 7 7 8 8 8 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 8 8 8 9 9 9 F F F 16 320-339 8 8 8 9 9 9 F F F 17 340-359 6 6 6 6 6 6 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 4 4 4 4 4 4 F F F 21 420-439 8 8 8 10 10 10 F F F 22 440-459 8 8 8 10 10 10 F F F 23 460-479 8 8 8 9 9 9 T T T 24 480-499 6 6 6 6 6 6 F F F 25 500-519 8 8 8 11 11 11 F F F 26 520-539 8 8 8 11 11 11 T T T 27 540-559 8 8 8 9 9 9 F F F 28 560-579 6 6 6 6 6 6 F F F 29 580-599 8 8 8 10 10 10 T T T Max gen: 8, Max flush: 11 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 8 8 8 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 9 9 9 12 12 12 F F F 3 060-079 4 4 4 3 3 3 F F F 4 080-099 5 5 5 5 5 5 F F F 5 100-119 6 6 6 7 7 7 F F F 6 120-139 8 8 8 11 11 11 F F F 7 140-159 7 7 7 8 8 8 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 6 6 6 6 6 6 F F F 10 200-219 8 8 8 9 9 9 F F F 11 220-239 8 8 8 10 10 10 F F F 12 240-259 7 7 7 8 8 8 F F F 13 260-279 7 7 7 8 8 8 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 8 8 8 9 9 9 F F F 16 320-339 8 8 8 9 9 9 F F F 17 340-359 6 6 6 6 6 6 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 4 4 4 4 4 4 F F F 21 420-439 8 8 8 10 10 10 F F F 22 440-459 8 8 8 10 10 10 F F F 23 460-479 9 9 9 12 12 12 F F F 24 480-499 6 6 6 6 6 6 F F F 25 500-519 8 8 8 11 11 11 F F F 26 520-539 9 9 9 12 12 12 F F F 27 540-559 8 8 8 9 9 9 F F F 28 560-579 6 6 6 6 6 6 F F F 29 580-599 9 9 9 12 12 12 F F F Max gen: 9, Max flush: 12 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.28168316Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.325059278Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.32572834Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.325765416Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.325780964Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.325796992Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227} {"msg":"Crucible 45805b09-5393-4877-a32a-c2e051052092 has session id: 44a48535-342c-4fbd-8aa9-7f6e6fca3e82","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.326186077Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Upstairs opts: Upstairs UUID: 45805b09-5393-4877-a32a-c2e051052092, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.326231631Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Crucible stats registered with UUID: 45805b09-5393-4877-a32a-c2e051052092","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.326269947Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356284055Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356344558Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227} {"msg":"45805b09-5393-4877-a32a-c2e051052092 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356375894Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.35641126Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"0","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356461994Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"0","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.35649894Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"1","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356532126Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"1","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356562672Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"2","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356594879Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"2","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356627145Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356660961Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356692197Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"ds_connection connected from Ok(127.0.0.1:49113)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356717134Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"ds_connection connected from Ok(127.0.0.1:46256)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356744871Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"ds_connection connected from Ok(127.0.0.1:59835)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.356931919Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.357565055Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"2","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.3576106Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"0","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.357713988Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"1","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.358192392Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"0","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.358234017Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"1","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.358320787Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"2","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.358822029Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.358849676Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"[0]R flush_numbers[0..12]: [8, 8, 12, 3, 5, 7, 11, 8, 1, 6, 9, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.358909309Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"[0]R generation[0..12]: [7, 7, 9, 4, 5, 6, 8, 7, 1, 6, 8, 8]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.358930206Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.358945034Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"[1]R flush_numbers[0..12]: [8, 8, 12, 3, 5, 7, 11, 8, 1, 6, 9, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.358958933Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"[1]R generation[0..12]: [7, 7, 9, 4, 5, 6, 8, 7, 1, 6, 8, 8]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.358972581Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.358986669Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"[2]R flush_numbers[0..12]: [8, 8, 12, 3, 5, 7, 11, 8, 1, 6, 9, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359000298Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"[2]R generation[0..12]: [7, 7, 9, 4, 5, 6, 8, 7, 1, 6, 8, 8]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359014096Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359028435Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359045862Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359060021Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359073199Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359086278Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"":"downstairs","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359099636Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO "2025-01-13T15:52:03.359111905Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"0","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359132603Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"1","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359145961Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"client":"2","":"downstairs","session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359159469Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359171918Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"45805b09-5393-4877-a32a-c2e051052092 is now active with session: 44a48535-342c-4fbd-8aa9-7f6e6fca3e82","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359185096Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359197235Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227,"session_id":"44a48535-342c-4fbd-8aa9-7f6e6fca3e82"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:03.359209754Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 153 len 40960 data: 3 2 2 2 2 2 2 2 2 2 02/30 Read block 19 len 28672 03/30 Read block 6 len 12288 04/30 Read block 269 len 24576 05/30 Read block 401 len 28672 06/30 Write block 570 len 12288 data: 3 2 2 07/30 Read block 573 len 36864 08/30 Flush 09/30 Write block 66 len 24576 data: 2 2 4 3 3 3 10/30 Read block 550 len 20480 11/30 Read block 445 len 20480 12/30 Read block 95 len 12288 13/30 Write block 119 len 8192 data: 2 2 14/30 Read block 581 len 16384 15/30 Write block 522 len 24576 data: 2 2 2 2 2 3 16/30 Write block 63 len 8192 data: 2 3 17/30 Flush 18/30 Write block 485 len 32768 data: 2 2 2 2 2 2 2 2 19/30 Read block 580 len 24576 20/30 Read block 234 len 12288 21/30 Read block 484 len 32768 22/30 Flush 23/30 Flush 24/30 Write block 590 len 40960 data: 2 2 2 2 2 2 2 2 2 2 25/30 Read block 68 len 24576 26/30 Read block 576 len 16384 27/30 Read block 450 len 12288 28/30 Read block 41 len 20480 29/30 Write block 223 len 40960 data: 2 2 2 2 2 2 2 3 2 2 30/30 Write block 332 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:03.380043792Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1227} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 8 8 8 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 9 9 9 12 12 12 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 5 5 5 5 5 5 F F F 5 100-119 10 10 10 14 14 14 F F F 6 120-139 10 10 10 14 14 14 F F F 7 140-159 10 10 10 13 13 13 F F F 8 160-179 10 10 10 13 13 13 F F F 9 180-199 6 6 6 6 6 6 F F F 10 200-219 8 8 8 9 9 9 F F F 11 220-239 8 8 8 10 10 10 T T T 12 240-259 7 7 7 8 8 8 F F F 13 260-279 7 7 7 8 8 8 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 8 8 8 9 9 9 F F F 16 320-339 8 8 8 9 9 9 T T T 17 340-359 6 6 6 6 6 6 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 4 4 4 4 4 4 F F F 21 420-439 8 8 8 10 10 10 F F F 22 440-459 8 8 8 10 10 10 F F F 23 460-479 9 9 9 12 12 12 F F F 24 480-499 10 10 10 15 15 15 F F F 25 500-519 8 8 8 11 11 11 F F F 26 520-539 10 10 10 14 14 14 F F F 27 540-559 8 8 8 9 9 9 F F F 28 560-579 10 10 10 13 13 13 F F F 29 580-599 9 9 9 12 12 12 T T T Max gen: 10, Max flush: 15 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 8 8 8 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 9 9 9 12 12 12 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 5 5 5 5 5 5 F F F 5 100-119 10 10 10 14 14 14 F F F 6 120-139 10 10 10 14 14 14 F F F 7 140-159 10 10 10 13 13 13 F F F 8 160-179 10 10 10 13 13 13 F F F 9 180-199 6 6 6 6 6 6 F F F 10 200-219 8 8 8 9 9 9 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 7 7 7 8 8 8 F F F 13 260-279 7 7 7 8 8 8 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 8 8 8 9 9 9 F F F 16 320-339 11 11 11 16 16 16 F F F 17 340-359 6 6 6 6 6 6 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 4 4 4 4 4 4 F F F 21 420-439 8 8 8 10 10 10 F F F 22 440-459 8 8 8 10 10 10 F F F 23 460-479 9 9 9 12 12 12 F F F 24 480-499 10 10 10 15 15 15 F F F 25 500-519 8 8 8 11 11 11 F F F 26 520-539 10 10 10 14 14 14 F F F 27 540-559 8 8 8 9 9 9 F F F 28 560-579 10 10 10 13 13 13 F F F 29 580-599 11 11 11 16 16 16 F F F Max gen: 11, Max flush: 16 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.639689841Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.682764454Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.683645301Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.683683727Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.683699825Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.683714404Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235} {"msg":"Crucible 3c2d118a-50b6-4f28-8172-231efc255a10 has session id: 726bf1f3-733d-4e61-a012-b381ae1c11b7","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.684147973Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Upstairs opts: Upstairs UUID: 3c2d118a-50b6-4f28-8172-231efc255a10, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.684193418Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Crucible stats registered with UUID: 3c2d118a-50b6-4f28-8172-231efc255a10","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.684218095Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.71519158Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.715243574Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235} {"msg":"3c2d118a-50b6-4f28-8172-231efc255a10 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.715261232Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.715277761Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"0","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.715308497Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"0","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.715326655Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"1","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.715344803Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"1","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.715361631Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"2","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.715379199Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"2","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.715469178Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.715511393Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.71553779Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:36104)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.715564307Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53321)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.715591574Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:38557)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.71562035Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.716520696Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"1","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.716617214Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"0","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.71665469Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"2","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.71716271Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"1","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.717203956Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"0","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.717315253Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"2","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.717751962Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.717837302Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"[0]R flush_numbers[0..12]: [8, 8, 12, 14, 5, 14, 14, 13, 13, 6, 9, 16]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.717963017Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"[0]R generation[0..12]: [7, 7, 9, 10, 5, 10, 10, 10, 10, 6, 8, 11]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.717982845Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.717997623Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"[1]R flush_numbers[0..12]: [8, 8, 12, 14, 5, 14, 14, 13, 13, 6, 9, 16]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718011791Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"[1]R generation[0..12]: [7, 7, 9, 10, 5, 10, 10, 10, 10, 6, 8, 11]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.71802571Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718039918Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"[2]R flush_numbers[0..12]: [8, 8, 12, 14, 5, 14, 14, 13, 13, 6, 9, 16]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718058356Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"[2]R generation[0..12]: [7, 7, 9, 10, 5, 10, 10, 10, 10, 6, 8, 11]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718072494Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718086413Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718099601Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.71811292Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718125788Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718138787Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"":"downstairs","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"Wait for a query_work_queue command to finish before sending IO msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718158784Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718171093Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"0","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718184701Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"1","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.71819817Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"client":"2","":"downstairs","session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718212468Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718224817Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"3c2d118a-50b6-4f28-8172-231efc255a10 is now active with session: 726bf1f3-733d-4e61-a012-b381ae1c11b7","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718237495Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718249534Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235,"session_id":"726bf1f3-733d-4e61-a012-b381ae1c11b7"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:07.718261712Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 336 len 16384 data: 2 2 2 2 02/30 Write block 503 len 16384 data: 4 3 4 4 03/30 Read block 144 len 12288 04/30 Read block 88 len 28672 05/30 Write block 500 len 12288 data: 5 4 4 06/30 Read block 346 len 40960 07/30 Read block 18 len 16384 08/30 Flush 09/30 Write block 265 len 28672 data: 3 4 4 4 4 4 4 10/30 Read block 353 len 36864 11/30 Read block 597 len 4096 12/30 Read block 309 len 20480 13/30 Write block 41 len 4096 data: 2 14/30 Flush 15/30 Write block 575 len 16384 data: 2 2 3 2 16/30 Write block 458 len 24576 data: 2 2 3 3 3 3 17/30 Write block 529 len 36864 data: 3 4 4 5 4 4 2 2 2 18/30 Read block 133 len 40960 19/30 Write block 116 len 4096 data: 3 20/30 Read block 108 len 36864 21/30 Read block 584 len 12288 22/30 Flush 23/30 Write block 580 len 8192 data: 3 3 24/30 Write block 550 len 16384 data: 4 4 4 4 25/30 Read block 242 len 8192 26/30 Read block 333 len 8192 27/30 Write block 354 len 40960 data: 2 2 2 2 2 2 2 2 2 2 28/30 Read block 315 len 32768 29/30 Write block 168 len 12288 data: 2 2 2 30/30 Read block 28 len 8192 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 8 6 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:07.736765193Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1235} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 8 8 8 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 12 12 12 18 18 18 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 5 5 5 5 5 5 F F F 5 100-119 12 12 12 19 19 19 F F F 6 120-139 10 10 10 14 14 14 F F F 7 140-159 10 10 10 13 13 13 F F F 8 160-179 10 10 10 13 13 13 T T T 9 180-199 6 6 6 6 6 6 F F F 10 200-219 8 8 8 9 9 9 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 7 7 7 8 8 8 F F F 13 260-279 12 12 12 18 18 18 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 8 8 8 9 9 9 F F F 16 320-339 12 12 12 17 17 17 F F F 17 340-359 6 6 6 6 6 6 T T T 18 360-379 1 1 1 1 1 1 T T T 19 380-399 7 7 7 8 8 8 F F F 20 400-419 4 4 4 4 4 4 F F F 21 420-439 8 8 8 10 10 10 F F F 22 440-459 12 12 12 19 19 19 F F F 23 460-479 12 12 12 19 19 19 F F F 24 480-499 10 10 10 15 15 15 F F F 25 500-519 12 12 12 17 17 17 F F F 26 520-539 12 12 12 19 19 19 F F F 27 540-559 8 8 8 9 9 9 T T T 28 560-579 12 12 12 19 19 19 F F F 29 580-599 11 11 11 16 16 16 T T T Max gen: 12, Max flush: 19 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 8 8 8 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 12 12 12 18 18 18 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 5 5 5 5 5 5 F F F 5 100-119 12 12 12 19 19 19 F F F 6 120-139 10 10 10 14 14 14 F F F 7 140-159 10 10 10 13 13 13 F F F 8 160-179 13 13 13 20 20 20 F F F 9 180-199 6 6 6 6 6 6 F F F 10 200-219 8 8 8 9 9 9 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 7 7 7 8 8 8 F F F 13 260-279 12 12 12 18 18 18 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 8 8 8 9 9 9 F F F 16 320-339 12 12 12 17 17 17 F F F 17 340-359 13 13 13 20 20 20 F F F 18 360-379 13 13 13 20 20 20 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 4 4 4 4 4 4 F F F 21 420-439 8 8 8 10 10 10 F F F 22 440-459 12 12 12 19 19 19 F F F 23 460-479 12 12 12 19 19 19 F F F 24 480-499 10 10 10 15 15 15 F F F 25 500-519 12 12 12 17 17 17 F F F 26 520-539 12 12 12 19 19 19 F F F 27 540-559 13 13 13 20 20 20 F F F 28 560-579 12 12 12 19 19 19 F F F 29 580-599 13 13 13 20 20 20 F F F Max gen: 13, Max flush: 20 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.009539668Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.056213582Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.057099329Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.057145483Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.057163431Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.057181449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243} {"msg":"Crucible eb7fdbb8-2641-4fb3-a805-4cc2bd091d89 has session id: 9731cbdd-03a3-4f62-ab8e-e89b88345428","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.057640655Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"Upstairs opts: Upstairs UUID: eb7fdbb8-2641-4fb3-a805-4cc2bd091d89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.057696089Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"Crucible stats registered with UUID: eb7fdbb8-2641-4fb3-a805-4cc2bd091d89","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.057728105Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088052517Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.08810819Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243} {"msg":"eb7fdbb8-2641-4fb3-a805-4cc2bd091d89 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088124478Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088141387Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"0","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088169643Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"0","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088187221Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"1","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088205019Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"1","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088221567Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"2","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088239175Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"2","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088255893Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088273171Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088290389Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"ds_connection connected from Ok(127.0.0.1:40761)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088447021Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"ds_connection connected from Ok(127.0.0.1:45860)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088499145Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"ds_connection connected from Ok(127.0.0.1:54439)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.088779862Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.089268085Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"0","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.0893099Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"2","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.089344196Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"1","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.089917569Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"0","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090010968Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"1","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090095108Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"2","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090525768Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090650684Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"[0]R flush_numbers[0..12]: [8, 8, 18, 14, 5, 19, 14, 13, 20, 6, 9, 16]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090781319Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"[0]R generation[0..12]: [7, 7, 12, 10, 5, 12, 10, 10, 13, 6, 8, 11]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090801886Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090817144Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"[1]R flush_numbers[0..12]: [8, 8, 18, 14, 5, 19, 14, 13, 20, 6, 9, 16]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090834672Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"[1]R generation[0..12]: [7, 7, 12, 10, 5, 12, 10, 10, 13, 6, 8, 11]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.09084964Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090863989Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"[2]R flush_numbers[0..12]: [8, 8, 18, 14, 5, 19, 14, 13, 20, 6, 9, 16]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090878077Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"[2]R generation[0..12]: [7, 7, 12, 10, 5, 12, 10, 10, 13, 6, 8, 11]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090891676Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090905724Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090919023Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090932471Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090952188Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090965447Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"":"downstairs","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090978856Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.090990934Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"0","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.091004502Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"1","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.091017961Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"client":"2","":"downstairs","session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.091031309Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.091047288Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"eb7fdbb8-2641-4fb3-a805-4cc2bd091d89 is now active with session: 9731cbdd-03a3-4f62-ab8e-e89b88345428","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.091060786Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.091073025Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243,"session_id":"9731cbdd-03a3-4f62-ab8e-e89b88345428"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:12.091085543Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 251 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Write block 186 len 36864 data: 3 2 3 3 2 2 2 2 2 03/30 Write block 328 len 20480 data: 2 2 2 2 3 04/30 Read block 166 len 32768 05/30 Read block 162 len 24576 06/30 Write block 579 len 28672 data: 2 4 4 3 4 3 3 07/30 Read block 253 len 8192 08/30 Write block 449 len 36864 data: 2 2 2 2 2 2 2 2 2 09/30 Read block 221 len 20480 10/30 Write block 40 len 16384 data: 2 3 2 4 11/30 Write block 248 len 32768 data: 2 2 2 3 3 3 3 3 12/30 Read block 25 len 4096 13/30 Read block 8 len 32768 14/30 Read block 130 len 24576 15/30 Read block 224 len 36864 16/30 Read block 240 len 24576 17/30 Write block 5 len 40960 data: 2 3 4 3 3 3 3 3 3 3 18/30 Read block 550 len 4096 19/30 Read block 208 len 24576 20/30 Read block 274 len 28672 21/30 Flush 22/30 Write block 416 len 36864 data: 3 3 3 3 3 2 2 2 2 23/30 Read block 542 len 32768 24/30 Read block 349 len 20480 25/30 Write block 527 len 28672 data: 4 3 4 5 5 6 5 26/30 Flush 27/30 Write block 137 len 32768 data: 3 3 3 4 4 4 3 3 28/30 Read block 230 len 36864 29/30 Read block 381 len 4096 30/30 Read block 423 len 20480 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:12.112730167Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1243} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 21 21 21 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 14 14 14 21 21 21 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 5 5 5 5 5 5 F F F 5 100-119 12 12 12 19 19 19 F F F 6 120-139 10 10 10 14 14 14 T T T 7 140-159 10 10 10 13 13 13 T T T 8 160-179 13 13 13 20 20 20 F F F 9 180-199 14 14 14 21 21 21 F F F 10 200-219 8 8 8 9 9 9 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 14 14 14 21 21 21 F F F 13 260-279 12 12 12 18 18 18 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 8 8 8 9 9 9 F F F 16 320-339 14 14 14 21 21 21 F F F 17 340-359 13 13 13 20 20 20 F F F 18 360-379 13 13 13 20 20 20 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 14 14 14 22 22 22 F F F 21 420-439 14 14 14 22 22 22 F F F 22 440-459 14 14 14 21 21 21 F F F 23 460-479 12 12 12 19 19 19 F F F 24 480-499 10 10 10 15 15 15 F F F 25 500-519 12 12 12 17 17 17 F F F 26 520-539 14 14 14 22 22 22 F F F 27 540-559 13 13 13 20 20 20 F F F 28 560-579 14 14 14 21 21 21 F F F 29 580-599 14 14 14 21 21 21 F F F Max gen: 14, Max flush: 22 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 21 21 21 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 14 14 14 21 21 21 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 5 5 5 5 5 5 F F F 5 100-119 12 12 12 19 19 19 F F F 6 120-139 15 15 15 23 23 23 F F F 7 140-159 15 15 15 23 23 23 F F F 8 160-179 13 13 13 20 20 20 F F F 9 180-199 14 14 14 21 21 21 F F F 10 200-219 8 8 8 9 9 9 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 14 14 14 21 21 21 F F F 13 260-279 12 12 12 18 18 18 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 8 8 8 9 9 9 F F F 16 320-339 14 14 14 21 21 21 F F F 17 340-359 13 13 13 20 20 20 F F F 18 360-379 13 13 13 20 20 20 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 14 14 14 22 22 22 F F F 21 420-439 14 14 14 22 22 22 F F F 22 440-459 14 14 14 21 21 21 F F F 23 460-479 12 12 12 19 19 19 F F F 24 480-499 10 10 10 15 15 15 F F F 25 500-519 12 12 12 17 17 17 F F F 26 520-539 14 14 14 22 22 22 F F F 27 540-559 13 13 13 20 20 20 F F F 28 560-579 14 14 14 21 21 21 F F F 29 580-599 14 14 14 21 21 21 F F F Max gen: 15, Max flush: 23 Begin loop 8 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.370758354Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.415307086Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.416110902Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.416161686Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.416186483Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.416207901Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251} {"msg":"Crucible cb724723-6599-4179-a7d2-af41631ef31e has session id: 7ef787bb-8205-483b-a9da-e8900b3c1e1b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.416733Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Upstairs opts: Upstairs UUID: cb724723-6599-4179-a7d2-af41631ef31e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.416778564Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Crucible stats registered with UUID: cb724723-6599-4179-a7d2-af41631ef31e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.416796112Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.447775127Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.447829021Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251} {"msg":"cb724723-6599-4179-a7d2-af41631ef31e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.447844579Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.447860507Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"0","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.447891154Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"0","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.447908992Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"1","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.447927049Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"1","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.447943748Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"2","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.447984463Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"2","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.448023178Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.448059754Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.448085791Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57175)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.448195528Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49693)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.448251142Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58850)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.448291087Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.449100963Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"0","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.449146237Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"2","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.449183203Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"1","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.449813549Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"0","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.449856575Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"2","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.449890071Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"1","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450431697Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450467723Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"[0]R flush_numbers[0..12]: [21, 8, 21, 14, 5, 19, 23, 23, 20, 21, 9, 16]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.45049746Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"[0]R generation[0..12]: [14, 7, 14, 10, 5, 12, 15, 15, 13, 14, 8, 11]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450520047Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450535565Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"[1]R flush_numbers[0..12]: [21, 8, 21, 14, 5, 19, 23, 23, 20, 21, 9, 16]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450549634Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"[1]R generation[0..12]: [14, 7, 14, 10, 5, 12, 15, 15, 13, 14, 8, 11]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450563522Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.45057807Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"[2]R flush_numbers[0..12]: [21, 8, 21, 14, 5, 19, 23, 23, 20, 21, 9, 16]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450591899Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"[2]R generation[0..12]: [14, 7, 14, 10, 5, 12, 15, 15, 13, 14, 8, 11]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450605297Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450619026Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450632014Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450645272Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.45066151Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450685518Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"downstairs","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"No downstairs reconciliation required","v":0,"name"Wait for a query_work_queue command to finish before sending IO :"crucible","level":30,"time":"2025-01-13T15:52:16.450718824Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450736652Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"0","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.45075382Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"1","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450767518Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"client":"2","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450781196Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450793605Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"cb724723-6599-4179-a7d2-af41631ef31e is now active with session: 7ef787bb-8205-483b-a9da-e8900b3c1e1b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450806533Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450818632Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.450831001Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 166 len 28672 data: 2 2 3 3 3 2 2 02/30 Read block 263 len 24576 03/30 Read block 474 len 12288 04/30 Read block 79 len 8192 05/30 Read block 222 len 28672 06/30 Read block 104 len 24576 07/30 Write block 548 len 8192 data: 4 4 08/30 Flush 09/30 Write block 320 len 24576 data: 3 3 3 3 2 2 10/30 Read block 383 len 16384 11/30 Write block 350 len 40960 data: 3 3 3 2 3 3 3 3 3 3 12/30 Read block 430 len 16384 13/30 Read block 106 len 20480 14/30 Read block 115 len 20480 15/30 Write block 115 len 40960 data: 3 4 2 2 3 3 3 3 3 3 16/30 Flush 17/30 Write block 314 len 12288 data: 2 2 2 18/30 Read block 513 len 4096 19/30 Read block 499 len 40960 20/30 Write block 102 len 8192 data: 2 2 21/30 Read block 397 len 8192 22/30 Read block 187 len 28672 23/30 Write block 474 len 28672 data: 3 3 3 3 4 3 2 24/30 Read block 403 len 16384 25/30 Read block 570 len 28672 26/30 Read block 504 len 16384 27/30 Write block 87 len 4096 data: 3 28/30 Write block 360 len 32768 data: 3 3 3 3 2 2 2 2 29/30 Read block 438 len 4096 30/30 Write block 355 len 40960 data: 4 4 4 4 4 4 4 4 4 3 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:16.471975043Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251} Wrote out file "/tmp/test_repair_verify.data" {"msg":"client task was cancelled without us; hopefully the program is exiting","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:16.472528729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"client task is sending Done(ReceiveTaskCancelled)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:16.472574424Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:16.472641246Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"7ef787bb-8205-483b-a9da-e8900b3c1e1b"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 21 21 21 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 14 14 14 21 21 21 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 5 5 5 5 5 5 T T T 5 100-119 16 16 16 25 25 25 T T T 6 120-139 16 16 16 25 25 25 F F F 7 140-159 15 15 15 23 23 23 F F F 8 160-179 16 16 16 24 24 24 F F F 9 180-199 14 14 14 21 21 21 F F F 10 200-219 8 8 8 9 9 9 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 14 14 14 21 21 21 F F F 13 260-279 12 12 12 18 18 18 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 8 8 8 9 9 9 T T T 16 320-339 16 16 16 25 25 25 F F F 17 340-359 16 16 16 25 25 25 T T T 18 360-379 13 13 13 20 20 20 T T T 19 380-399 7 7 7 8 8 8 F F F 20 400-419 14 14 14 22 22 22 F F F 21 420-439 14 14 14 22 22 22 F F F 22 440-459 14 14 14 21 21 21 F F F 23 460-479 12 12 12 19 19 19 T T T 24 480-499 10 10 10 15 15 15 T T T 25 500-519 12 12 12 17 17 17 F F F 26 520-539 14 14 14 22 22 22 F F F 27 540-559 16 16 16 24 24 24 F F F 28 560-579 14 14 14 21 21 21 F F F 29 580-599 14 14 14 21 21 21 F F F Max gen: 16, Max flush: 25 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 21 21 21 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 14 14 14 21 21 21 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 17 17 17 26 26 26 F F F 5 100-119 17 17 17 26 26 26 F F F 6 120-139 16 16 16 25 25 25 F F F 7 140-159 15 15 15 23 23 23 F F F 8 160-179 16 16 16 24 24 24 F F F 9 180-199 14 14 14 21 21 21 F F F 10 200-219 8 8 8 9 9 9 F F F 11 220-239 11 11 11 16 16 16 F F F 12 240-259 14 14 14 21 21 21 F F F 13 260-279 12 12 12 18 18 18 F F F 14 280-299 4 4 4 3 3 3 F F F 15 300-319 17 17 17 26 26 26 F F F 16 320-339 16 16 16 25 25 25 F F F 17 340-359 17 17 17 26 26 26 F F F 18 360-379 17 17 17 26 26 26 F F F 19 380-399 7 7 7 8 8 8 F F F 20 400-419 14 14 14 22 22 22 F F F 21 420-439 14 14 14 22 22 22 F F F 22 440-459 14 14 14 21 21 21 F F F 23 460-479 17 17 17 26 26 26 F F F 24 480-499 17 17 17 26 26 26 F F F 25 500-519 12 12 12 17 17 17 F F F 26 520-539 14 14 14 22 22 22 F F F 27 540-559 16 16 16 24 24 24 F F F 28 560-579 14 14 14 21 21 21 F F F 29 580-599 14 14 14 21 21 21 F F F Max gen: 17, Max flush: 26 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.764563056Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.810826797Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.811705775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.811754279Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.811772717Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.811791595Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259} {"msg":"Crucible 54cf6f82-efc2-4488-ab8d-c16034f64b10 has session id: caa09fb2-f5a4-4c52-b849-84e3fd8609e4","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.812312014Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Upstairs opts: Upstairs UUID: 54cf6f82-efc2-4488-ab8d-c16034f64b10, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.812357659Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Crucible stats registered with UUID: 54cf6f82-efc2-4488-ab8d-c16034f64b10","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.812374857Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.842985195Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843045158Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259} {"msg":"54cf6f82-efc2-4488-ab8d-c16034f64b10 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843071605Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843088643Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"0","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843119389Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"0","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843137417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"1","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843155165Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"1","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843245574Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"2","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843394327Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"2","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843430973Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843815668Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843853794Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42768)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843871312Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35313)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.84388663Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:50483)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.843902228Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.844391671Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"0","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.844419688Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"1","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.844554572Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"2","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.844986991Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"0","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.845114646Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"1","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.8452502Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"2","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.845530098Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.845642135Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"[0]R flush_numbers[0..12]: [21, 8, 21, 14, 26, 26, 25, 23, 24, 21, 9, 16]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.845914503Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"[0]R generation[0..12]: [14, 7, 14, 10, 17, 17, 16, 15, 16, 14, 8, 11]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.845933951Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.845948599Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"[1]R flush_numbers[0..12]: [21, 8, 21, 14, 26, 26, 25, 23, 24, 21, 9, 16]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.845962747Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"[1]R generation[0..12]: [14, 7, 14, 10, 17, 17, 16, 15, 16, 14, 8, 11]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.845976556Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846003613Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"[2]R flush_numbers[0..12]: [21, 8, 21, 14, 26, 26, 25, 23, 24, 21, 9, 16]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846019231Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"[2]R generation[0..12]: [14, 7, 14, 10, 17, 17, 16, 15, 16, 14, 8, 11]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846033229Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846051007Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846064376Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846077624Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846090573Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":Wait for a query_work_queue command to finish before sending IO "All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.84611233Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"":"downstairs","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846137547Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846162674Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"0","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846189551Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"1","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846216498Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"client":"2","":"downstairs","session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846243795Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846269972Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"54cf6f82-efc2-4488-ab8d-c16034f64b10 is now active with session: caa09fb2-f5a4-4c52-b849-84e3fd8609e4","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846297498Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259,"session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846323805Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"caa09fb2-f5a4-4c52-b849-84e3fd8609e4"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:20.846354642Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 299 len 8192 data: 2 2 02/30 Read block 545 len 12288 03/30 Write block 528 len 20480 data: 4 5 6 6 7 04/30 Read block 282 len 4096 05/30 Read block 572 len 4096 06/30 Read block 128 len 28672 07/30 Read block 325 len 8192 08/30 Read block 0 len 16384 09/30 Write block 444 len 40960 data: 2 2 2 2 2 3 3 3 3 3 10/30 Read block 321 len 20480 11/30 Read block 397 len 8192 12/30 Write block 514 len 28672 data: 3 3 4 4 3 3 2 13/30 Read block 334 len 12288 14/30 Write block 279 len 16384 data: 2 2 2 2 15/30 Write block 123 len 20480 data: 4 4 3 4 4 16/30 Write block 277 len 16384 data: 3 2 3 3 17/30 Write block 381 len 4096 data: 3 18/30 Write block 335 len 8192 data: 2 3 19/30 Write block 176 len 20480 data: 2 2 2 2 2 20/30 Read block 565 len 16384 21/30 Write block 216 len 40960 data: 2 3 3 2 2 2 2 3 3 3 22/30 Read block 153 len 4096 23/30 Read block 325 len 24576 24/30 Read block 418 len 32768 25/30 Write block 400 len 36864 data: 2 2 2 2 2 2 2 2 2 26/30 Write block 467 len 8192 data: 2 2 27/30 Write block 260 len 24576 data: 2 2 2 2 3 4 28/30 Read block 287 len 16384 29/30 Read block 507 len 12288 30/30 Read block 90 len 12288 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:20.864857163Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1259} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 21 21 21 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 14 14 14 21 21 21 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 17 17 17 26 26 26 F F F 5 100-119 17 17 17 26 26 26 F F F 6 120-139 16 16 16 25 25 25 T T T 7 140-159 15 15 15 23 23 23 F F F 8 160-179 16 16 16 24 24 24 T T T 9 180-199 14 14 14 21 21 21 T T T 10 200-219 8 8 8 9 9 9 T T T 11 220-239 11 11 11 16 16 16 T T T 12 240-259 14 14 14 21 21 21 F F F 13 260-279 12 12 12 18 18 18 T T T 14 280-299 4 4 4 3 3 3 T T T 15 300-319 17 17 17 26 26 26 T T T 16 320-339 16 16 16 25 25 25 T T T 17 340-359 17 17 17 26 26 26 F F F 18 360-379 17 17 17 26 26 26 F F F 19 380-399 7 7 7 8 8 8 T T T 20 400-419 14 14 14 22 22 22 T T T 21 420-439 14 14 14 22 22 22 F F F 22 440-459 14 14 14 21 21 21 T T T 23 460-479 17 17 17 26 26 26 T T T 24 480-499 17 17 17 26 26 26 F F F 25 500-519 12 12 12 17 17 17 T T T 26 520-539 14 14 14 22 22 22 T T T 27 540-559 16 16 16 24 24 24 F F F 28 560-579 14 14 14 21 21 21 F F F 29 580-599 14 14 14 21 21 21 F F F Max gen: 17, Max flush: 26 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 21 21 21 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 14 14 14 21 21 21 F F F 3 060-079 10 10 10 14 14 14 F F F 4 080-099 17 17 17 26 26 26 F F F 5 100-119 17 17 17 26 26 26 F F F 6 120-139 18 18 18 27 27 27 F F F 7 140-159 15 15 15 23 23 23 F F F 8 160-179 18 18 18 27 27 27 F F F 9 180-199 18 18 18 27 27 27 F F F 10 200-219 18 18 18 27 27 27 F F F 11 220-239 18 18 18 27 27 27 F F F 12 240-259 14 14 14 21 21 21 F F F 13 260-279 18 18 18 27 27 27 F F F 14 280-299 18 18 18 27 27 27 F F F 15 300-319 18 18 18 27 27 27 F F F 16 320-339 18 18 18 27 27 27 F F F 17 340-359 17 17 17 26 26 26 F F F 18 360-379 17 17 17 26 26 26 F F F 19 380-399 18 18 18 27 27 27 F F F 20 400-419 18 18 18 27 27 27 F F F 21 420-439 14 14 14 22 22 22 F F F 22 440-459 18 18 18 27 27 27 F F F 23 460-479 18 18 18 27 27 27 F F F 24 480-499 17 17 17 26 26 26 F F F 25 500-519 18 18 18 27 27 27 F F F 26 520-539 18 18 18 27 27 27 F F F 27 540-559 16 16 16 24 24 24 F F F 28 560-579 14 14 14 21 21 21 F F F 29 580-599 14 14 14 21 21 21 F F F Max gen: 18, Max flush: 27 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.198945454Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.242468975Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.243129988Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.243176553Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.24319901Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.243217398Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267} {"msg":"Crucible 943cc0fa-bc31-4df1-ac07-7212da844863 has session id: 958b875f-a6eb-461f-99ee-2cd7275f59af","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.24362923Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Upstairs opts: Upstairs UUID: 943cc0fa-bc31-4df1-ac07-7212da844863, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.243677444Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Crucible stats registered with UUID: 943cc0fa-bc31-4df1-ac07-7212da844863","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.243702581Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.273769203Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.273804548Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267} {"msg":"943cc0fa-bc31-4df1-ac07-7212da844863 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.273838465Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.273855393Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"0","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.273883099Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"0","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.273900707Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"1","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.273918635Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"1","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.273935293Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"2","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.273952931Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"2","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.273975209Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.273993126Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.274010195Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"ds_connection connected from Ok(127.0.0.1:37894)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.274040511Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"ds_connection connected from Ok(127.0.0.1:41394)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.274294211Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"ds_connection connected from Ok(127.0.0.1:56650)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.274335617Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.274745759Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"0","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.274776475Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"2","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.275037475Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"1","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.275498321Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"2","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.275527547Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"0","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.275542966Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"1","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276044657Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276072434Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"[0]R flush_numbers[0..12]: [21, 8, 21, 14, 26, 26, 27, 23, 27, 27, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.27610208Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"[0]R generation[0..12]: [14, 7, 14, 10, 17, 17, 18, 15, 18, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276119029Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276133817Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"[1]R flush_numbers[0..12]: [21, 8, 21, 14, 26, 26, 27, 23, 27, 27, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276148055Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"[1]R generation[0..12]: [14, 7, 14, 10, 17, 17, 18, 15, 18, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276161733Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276175602Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"[2]R flush_numbers[0..12]: [21, 8, 21, 14, 26, 26, 27, 23, 27, 27, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.27618921Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"[2]R generation[0..12]: [14, 7, 14, 10, 17, 17, 18, 15, 18, 18, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276202799Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276216757Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Max found gen is 19","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276230156Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Generation requested: 20 >= found:19","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276243524Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276256482Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276269351Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"":"downstairs","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.27628255Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276344652Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"0","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.27636456Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"1","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276378188Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"client":"2","":"downstairs","session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276391617Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276403945Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"943cc0fa-bc31-4df1-ac07-7212da844863 is now active with session: 958b875f-a6eb-461f-99ee-2cd7275f59af","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276416704Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276428762Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267,"session_id":"958b875f-a6eb-461f-99ee-2cd7275f59af"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:25.276440861Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 91 len 28672 data: 3 3 3 3 3 2 2 02/30 Read block 498 len 16384 03/30 Read block 445 len 20480 04/30 Read block 303 len 28672 05/30 Read block 77 len 36864 06/30 Read block 465 len 4096 07/30 Read block 432 len 36864 08/30 Write block 193 len 8192 data: 3 3 09/30 Flush 10/30 Flush 11/30 Write block 184 len 36864 data: 3 3 4 3 4 4 3 3 3 12/30 Read block 334 len 4096 13/30 Write block 69 len 16384 data: 4 4 4 3 14/30 Write block 274 len 40960 data: 3 3 3 4 3 4 4 3 3 2 15/30 Read block 69 len 32768 16/30 Read block 543 len 12288 17/30 Write block 81 len 36864 data: 2 2 2 2 2 3 4 3 3 18/30 Read block 233 len 4096 19/30 Flush 20/30 Write block 46 len 40960 data: 4 3 3 3 5 5 5 4 4 4 21/30 Write block 405 len 40960 data: 3 3 3 3 2 3 2 2 3 3 22/30 Read block 188 len 8192 23/30 Read block 274 len 24576 24/30 Flush 25/30 Write block 383 len 28672 data: 3 3 3 3 3 3 3 26/30 Read block 442 len 40960 27/30 Write block 316 len 4096 data: 3 28/30 Read block 223 len 40960 29/30 Write block 104 len 8192 data: 3 3 30/30 Write block 551 len 36864 data: 5 5 5 3 4 4 4 4 3 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:25.297505093Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1267} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 21 21 21 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 20 20 20 31 31 31 F F F 3 060-079 20 20 20 30 30 30 F F F 4 080-099 20 20 20 30 30 30 F F F 5 100-119 17 17 17 26 26 26 T T T 6 120-139 18 18 18 27 27 27 F F F 7 140-159 15 15 15 23 23 23 F F F 8 160-179 18 18 18 27 27 27 F F F 9 180-199 20 20 20 30 30 30 F F F 10 200-219 18 18 18 27 27 27 F F F 11 220-239 18 18 18 27 27 27 F F F 12 240-259 14 14 14 21 21 21 F F F 13 260-279 20 20 20 30 30 30 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 18 18 18 27 27 27 T T T 16 320-339 18 18 18 27 27 27 F F F 17 340-359 17 17 17 26 26 26 F F F 18 360-379 17 17 17 26 26 26 F F F 19 380-399 18 18 18 27 27 27 T T T 20 400-419 20 20 20 31 31 31 F F F 21 420-439 14 14 14 22 22 22 F F F 22 440-459 18 18 18 27 27 27 F F F 23 460-479 18 18 18 27 27 27 F F F 24 480-499 17 17 17 26 26 26 F F F 25 500-519 18 18 18 27 27 27 F F F 26 520-539 18 18 18 27 27 27 F F F 27 540-559 16 16 16 24 24 24 T T T 28 560-579 14 14 14 21 21 21 F F F 29 580-599 14 14 14 21 21 21 F F F Max gen: 20, Max flush: 31 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 21 21 21 F F F 1 020-039 7 7 7 8 8 8 F F F 2 040-059 20 20 20 31 31 31 F F F 3 060-079 20 20 20 30 30 30 F F F 4 080-099 20 20 20 30 30 30 F F F 5 100-119 21 21 21 32 32 32 F F F 6 120-139 18 18 18 27 27 27 F F F 7 140-159 15 15 15 23 23 23 F F F 8 160-179 18 18 18 27 27 27 F F F 9 180-199 20 20 20 30 30 30 F F F 10 200-219 18 18 18 27 27 27 F F F 11 220-239 18 18 18 27 27 27 F F F 12 240-259 14 14 14 21 21 21 F F F 13 260-279 20 20 20 30 30 30 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 21 21 21 32 32 32 F F F 16 320-339 18 18 18 27 27 27 F F F 17 340-359 17 17 17 26 26 26 F F F 18 360-379 17 17 17 26 26 26 F F F 19 380-399 21 21 21 32 32 32 F F F 20 400-419 20 20 20 31 31 31 F F F 21 420-439 14 14 14 22 22 22 F F F 22 440-459 18 18 18 27 27 27 F F F 23 460-479 18 18 18 27 27 27 F F F 24 480-499 17 17 17 26 26 26 F F F 25 500-519 18 18 18 27 27 27 F F F 26 520-539 18 18 18 27 27 27 F F F 27 540-559 21 21 21 32 32 32 F F F 28 560-579 14 14 14 21 21 21 F F F 29 580-599 14 14 14 21 21 21 F F F Max gen: 21, Max flush: 32 Begin loop 11 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.565781825Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.609595872Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.610501836Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.610541752Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.61055628Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.610571418Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275} {"msg":"Crucible 965614f0-965e-491d-8b29-2a72418b3ba9 has session id: c6f58f20-f129-4fae-bf96-797add1b868f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.611104826Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Upstairs opts: Upstairs UUID: 965614f0-965e-491d-8b29-2a72418b3ba9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.611147241Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Crucible stats registered with UUID: 965614f0-965e-491d-8b29-2a72418b3ba9","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.611174128Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.641742011Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.641792995Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275} {"msg":"965614f0-965e-491d-8b29-2a72418b3ba9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.641817202Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.641833531Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"0","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.641861727Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"0","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.641879335Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"1","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.641900863Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"1","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.641915081Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"2","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.641929509Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"2","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.641942768Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.641965135Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.642019559Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58419)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.642127786Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41679)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.642162762Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33551)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.642237313Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.642999564Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"2","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.64303666Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"0","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.643245856Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"1","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.643770114Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"0","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.643848855Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"2","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.643886651Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"1","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644351427Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644457464Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"[0]R flush_numbers[0..12]: [21, 8, 31, 30, 30, 32, 27, 23, 27, 30, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644587299Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"[0]R generation[0..12]: [14, 7, 20, 20, 20, 21, 18, 15, 18, 20, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644607387Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644622085Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"[1]R flush_numbers[0..12]: [21, 8, 31, 30, 30, 32, 27, 23, 27, 30, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644636124Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"[1]R generation[0..12]: [14, 7, 20, 20, 20, 21, 18, 15, 18, 20, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644650222Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.64466432Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"[2]R flush_numbers[0..12]: [21, 8, 31, 30, 30, 32, 27, 23, 27, 30, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644678208Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"[2]R generation[0..12]: [14, 7, 20, 20, 20, 21, 18, 15, 18, 20, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644691617Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644706495Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644719524Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644733242Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644746211Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644763229Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"":"downstairs","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644776837Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:52:29.644788856Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"0","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644806324Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"1","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644819562Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"client":"2","":"downstairs","session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644832761Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644845069Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"965614f0-965e-491d-8b29-2a72418b3ba9 is now active with session: c6f58f20-f129-4fae-bf96-797add1b868f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644857687Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644870346Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275,"session_id":"c6f58f20-f129-4fae-bf96-797add1b868f"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:29.644882635Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 13 len 36864 data: 4 4 2 4 4 4 4 4 2 02/30 Read block 494 len 4096 03/30 Write block 380 len 4096 data: 2 04/30 Read block 523 len 24576 05/30 Read block 208 len 20480 06/30 Write block 539 len 36864 data: 2 2 2 2 2 2 2 3 3 07/30 Read block 438 len 32768 08/30 Read block 314 len 16384 09/30 Write block 82 len 20480 data: 3 3 3 3 4 10/30 Write block 232 len 4096 data: 3 11/30 Read block 163 len 40960 12/30 Write block 228 len 20480 data: 3 3 4 3 4 13/30 Write block 516 len 20480 data: 5 5 4 4 3 14/30 Write block 421 len 8192 data: 3 3 15/30 Read block 323 len 36864 16/30 Read block 502 len 36864 17/30 Read block 435 len 8192 18/30 Write block 591 len 16384 data: 3 3 3 3 19/30 Write block 506 len 12288 data: 5 4 4 20/30 Read block 519 len 12288 21/30 Read block 565 len 12288 22/30 Read block 261 len 24576 23/30 Write block 91 len 20480 data: 4 4 4 4 4 24/30 Read block 515 len 4096 25/30 Write block 186 len 24576 data: 5 4 5 5 4 4 26/30 Read block 142 len 32768 27/30 Write block 85 len 28672 data: 4 5 5 4 4 3 5 28/30 Read block 403 len 16384 29/30 Write block 443 len 4096 data: 3 30/30 Read block 509 len 28672 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:29.664268802Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1275} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 21 21 21 T T T 1 020-039 7 7 7 8 8 8 T T T 2 040-059 20 20 20 31 31 31 F F F 3 060-079 20 20 20 30 30 30 F F F 4 080-099 20 20 20 30 30 30 T T T 5 100-119 21 21 21 32 32 32 F F F 6 120-139 18 18 18 27 27 27 F F F 7 140-159 15 15 15 23 23 23 F F F 8 160-179 18 18 18 27 27 27 F F F 9 180-199 20 20 20 30 30 30 T T T 10 200-219 18 18 18 27 27 27 F F F 11 220-239 18 18 18 27 27 27 T T T 12 240-259 14 14 14 21 21 21 F F F 13 260-279 20 20 20 30 30 30 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 21 21 21 32 32 32 F F F 16 320-339 18 18 18 27 27 27 F F F 17 340-359 17 17 17 26 26 26 F F F 18 360-379 17 17 17 26 26 26 F F F 19 380-399 21 21 21 32 32 32 T T T 20 400-419 20 20 20 31 31 31 F F F 21 420-439 14 14 14 22 22 22 T T T 22 440-459 18 18 18 27 27 27 T T T 23 460-479 18 18 18 27 27 27 F F F 24 480-499 17 17 17 26 26 26 F F F 25 500-519 18 18 18 27 27 27 T T T 26 520-539 18 18 18 27 27 27 T T T 27 540-559 21 21 21 32 32 32 T T T 28 560-579 14 14 14 21 21 21 F F F 29 580-599 14 14 14 21 21 21 T T T Max gen: 21, Max flush: 32 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 33 33 33 F F F 1 020-039 22 22 22 33 33 33 F F F 2 040-059 20 20 20 31 31 31 F F F 3 060-079 20 20 20 30 30 30 F F F 4 080-099 22 22 22 33 33 33 F F F 5 100-119 21 21 21 32 32 32 F F F 6 120-139 18 18 18 27 27 27 F F F 7 140-159 15 15 15 23 23 23 F F F 8 160-179 18 18 18 27 27 27 F F F 9 180-199 22 22 22 33 33 33 F F F 10 200-219 18 18 18 27 27 27 F F F 11 220-239 22 22 22 33 33 33 F F F 12 240-259 14 14 14 21 21 21 F F F 13 260-279 20 20 20 30 30 30 F F F 14 280-299 20 20 20 30 30 30 F F F 15 300-319 21 21 21 32 32 32 F F F 16 320-339 18 18 18 27 27 27 F F F 17 340-359 17 17 17 26 26 26 F F F 18 360-379 17 17 17 26 26 26 F F F 19 380-399 22 22 22 33 33 33 F F F 20 400-419 20 20 20 31 31 31 F F F 21 420-439 22 22 22 33 33 33 F F F 22 440-459 22 22 22 33 33 33 F F F 23 460-479 18 18 18 27 27 27 F F F 24 480-499 17 17 17 26 26 26 F F F 25 500-519 22 22 22 33 33 33 F F F 26 520-539 22 22 22 33 33 33 F F F 27 540-559 22 22 22 33 33 33 F F F 28 560-579 14 14 14 21 21 21 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 22, Max flush: 33 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:33.980920287Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.025074905Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.025728309Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.025773513Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.025792801Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.025810629Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283} {"msg":"Crucible b867346e-04ca-49ff-b7b0-d857c7ddfe40 has session id: 676f8d35-ce3b-4518-a0c7-b817d677f467","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.02631654Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Upstairs opts: Upstairs UUID: b867346e-04ca-49ff-b7b0-d857c7ddfe40, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.026368064Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Crucible stats registered with UUID: b867346e-04ca-49ff-b7b0-d857c7ddfe40","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.026391871Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056037812Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056080787Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283} {"msg":"b867346e-04ca-49ff-b7b0-d857c7ddfe40 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056106544Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056123412Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"0","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056152679Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"0","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056171896Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"1","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056189824Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"1","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056206322Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"2","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.05622417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"2","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056244468Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.05630839Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056359175Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"ds_connection connected from Ok(127.0.0.1:63314)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056499338Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"ds_connection connected from Ok(127.0.0.1:45826)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056548582Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"ds_connection connected from Ok(127.0.0.1:60347)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.056629183Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.057042275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"2","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.057208375Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"0","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.057328461Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"1","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.057734614Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"2","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.057778009Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"0","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.057946659Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"1","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058352902Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058480367Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"[0]R flush_numbers[0..12]: [33, 33, 31, 30, 33, 32, 27, 23, 27, 33, 27, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058516883Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"[0]R generation[0..12]: [22, 22, 20, 20, 22, 21, 18, 15, 18, 22, 18, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058532851Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058547949Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"[1]R flush_numbers[0..12]: [33, 33, 31, 30, 33, 32, 27, 23, 27, 33, 27, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058562267Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"[1]R generation[0..12]: [22, 22, 20, 20, 22, 21, 18, 15, 18, 22, 18, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058579275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058593714Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"[2]R flush_numbers[0..12]: [33, 33, 31, 30, 33, 32, 27, 23, 27, 33, 27, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058607702Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"[2]R generation[0..12]: [22, 22, 20, 20, 22, 21, 18, 15, 18, 22, 18, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058621511Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058635709Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Max found gen is 23","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058648987Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Generation requested: 24 >= found:23","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058662466Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058675614Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":"downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058688983Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"":Wait for a query_work_queue command to finish before sending IO "downstairs","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.05871512Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058740346Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"0","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058766954Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"1","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.0587928Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"client":"2","":"downstairs","session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058811238Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058824167Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"b867346e-04ca-49ff-b7b0-d857c7ddfe40 is now active with session: 676f8d35-ce3b-4518-a0c7-b817d677f467","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058839565Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283,"session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058851933Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "pid":1283,"session_id":"676f8d35-ce3b-4518-a0c7-b817d677f467"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:34.058868262Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 292 len 4096 data: 4 02/30 Read block 357 len 24576 03/30 Write block 259 len 36864 data: 3 3 3 3 3 4 5 5 5 04/30 Read block 249 len 8192 05/30 Read block 333 len 36864 06/30 Write block 98 len 32768 data: 2 2 2 2 3 3 4 4 07/30 Read block 32 len 36864 08/30 Write block 534 len 16384 data: 5 3 3 3 09/30 Read block 303 len 4096 10/30 Flush 11/30 Write block 365 len 16384 data: 3 3 3 2 12/30 Write block 161 len 40960 data: 3 3 2 2 2 3 3 4 4 4 13/30 Flush 14/30 Write block 248 len 40960 data: 3 3 3 4 4 4 4 4 3 3 15/30 Read block 132 len 8192 16/30 Write block 207 len 40960 data: 2 2 2 2 2 2 2 2 2 3 17/30 Read block 182 len 12288 18/30 Flush 19/30 Write block 413 len 4096 data: 4 20/30 Write block 31 len 12288 data: 3 3 3 21/30 Read block 251 len 28672 22/30 Read block 381 len 40960 23/30 Read block 32 len 8192 24/30 Write block 125 len 12288 data: 4 5 5 25/30 Read block 100 len 8192 26/30 Read block 186 len 8192 27/30 Read block 76 len 40960 28/30 Read block 351 len 36864 29/30 Read block 322 len 24576 30/30 Write block 485 len 28672 data: 3 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 1 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:34.079167953Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1283} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 33 33 33 F F F 1 020-039 22 22 22 33 33 33 T T T 2 040-059 20 20 20 31 31 31 F F F 3 060-079 20 20 20 30 30 30 F F F 4 080-099 24 24 24 34 34 34 F F F 5 100-119 24 24 24 34 34 34 F F F 6 120-139 18 18 18 27 27 27 T T T 7 140-159 15 15 15 23 23 23 F F F 8 160-179 24 24 24 35 35 35 F F F 9 180-199 22 22 22 33 33 33 F F F 10 200-219 24 24 24 36 36 36 F F F 11 220-239 22 22 22 33 33 33 F F F 12 240-259 24 24 24 36 36 36 F F F 13 260-279 24 24 24 34 34 34 F F F 14 280-299 24 24 24 34 34 34 F F F 15 300-319 21 21 21 32 32 32 F F F 16 320-339 18 18 18 27 27 27 F F F 17 340-359 17 17 17 26 26 26 F F F 18 360-379 24 24 24 35 35 35 F F F 19 380-399 22 22 22 33 33 33 F F F 20 400-419 20 20 20 31 31 31 T T T 21 420-439 22 22 22 33 33 33 F F F 22 440-459 22 22 22 33 33 33 F F F 23 460-479 18 18 18 27 27 27 F F F 24 480-499 17 17 17 26 26 26 T T T 25 500-519 22 22 22 33 33 33 F F F 26 520-539 24 24 24 34 34 34 F F F 27 540-559 22 22 22 33 33 33 F F F 28 560-579 14 14 14 21 21 21 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 24, Max flush: 36 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 33 33 33 F F F 1 020-039 25 25 25 37 37 37 F F F 2 040-059 20 20 20 31 31 31 F F F 3 060-079 20 20 20 30 30 30 F F F 4 080-099 24 24 24 34 34 34 F F F 5 100-119 24 24 24 34 34 34 F F F 6 120-139 25 25 25 37 37 37 F F F 7 140-159 15 15 15 23 23 23 F F F 8 160-179 24 24 24 35 35 35 F F F 9 180-199 22 22 22 33 33 33 F F F 10 200-219 24 24 24 36 36 36 F F F 11 220-239 22 22 22 33 33 33 F F F 12 240-259 24 24 24 36 36 36 F F F 13 260-279 24 24 24 34 34 34 F F F 14 280-299 24 24 24 34 34 34 F F F 15 300-319 21 21 21 32 32 32 F F F 16 320-339 18 18 18 27 27 27 F F F 17 340-359 17 17 17 26 26 26 F F F 18 360-379 24 24 24 35 35 35 F F F 19 380-399 22 22 22 33 33 33 F F F 20 400-419 25 25 25 37 37 37 F F F 21 420-439 22 22 22 33 33 33 F F F 22 440-459 22 22 22 33 33 33 F F F 23 460-479 18 18 18 27 27 27 F F F 24 480-499 25 25 25 37 37 37 F F F 25 500-519 22 22 22 33 33 33 F F F 26 520-539 24 24 24 34 34 34 F F F 27 540-559 22 22 22 33 33 33 F F F 28 560-579 14 14 14 21 21 21 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 25, Max flush: 37 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.351741231Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.396566681Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.39751664Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.397564644Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.397587952Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.397606619Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291} {"msg":"Crucible e1279db4-e22a-40b9-b167-110fc64a558b has session id: 456f17a3-f9ce-4f82-ab26-374bb6baac76","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.398097842Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Upstairs opts: Upstairs UUID: e1279db4-e22a-40b9-b167-110fc64a558b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.398143447Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Crucible stats registered with UUID: e1279db4-e22a-40b9-b167-110fc64a558b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.398168424Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.429509297Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.42957047Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291} {"msg":"e1279db4-e22a-40b9-b167-110fc64a558b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.429589467Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.429605756Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"0","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.429637152Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"0","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.42965477Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"1","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.429672688Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"1","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.429689496Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"2","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.429707694Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"2","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.429731181Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.429749169Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.429766357Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"ds_connection connected from Ok(127.0.0.1:57451)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.430037895Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"ds_connection connected from Ok(127.0.0.1:49734)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.430113106Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"ds_connection connected from Ok(127.0.0.1:60547)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.430147622Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.430753452Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"0","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.430949169Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"2","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.431135127Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"1","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.431489376Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"0","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.431519112Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"2","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.431764644Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"1","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432048751Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432187544Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"[0]R flush_numbers[0..12]: [33, 37, 31, 30, 34, 34, 37, 23, 35, 33, 36, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432422567Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"[0]R generation[0..12]: [22, 25, 20, 20, 24, 24, 25, 15, 24, 22, 24, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432444994Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432460132Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"[1]R flush_numbers[0..12]: [33, 37, 31, 30, 34, 34, 37, 23, 35, 33, 36, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432474151Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"[1]R generation[0..12]: [22, 25, 20, 20, 24, 24, 25, 15, 24, 22, 24, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432487999Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432501968Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"[2]R flush_numbers[0..12]: [33, 37, 31, 30, 34, 34, 37, 23, 35, 33, 36, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432515766Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"[2]R generation[0..12]: [22, 25, 20, 20, 24, 24, 25, 15, 24, 22, 24, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432529504Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432543353Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432556471Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.43256979Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Next flush: 38","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-01-13T15:52:38.432582718Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432602516Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"":"downstairs","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432615764Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432627963Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"0","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432644171Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"1","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432657729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"client":"2","":"downstairs","session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432671078Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432683497Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"e1279db4-e22a-40b9-b167-110fc64a558b is now active with session: 456f17a3-f9ce-4f82-ab26-374bb6baac76","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432696415Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432708443Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291,"session_id":"456f17a3-f9ce-4f82-ab26-374bb6baac76"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:38.432720752Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 122 len 32768 data: 4 5 5 5 6 6 4 3 02/30 Write block 225 len 36864 data: 4 3 3 4 4 5 4 5 2 03/30 Flush 04/30 Write block 359 len 8192 data: 5 5 05/30 Write block 18 len 20480 data: 5 5 5 3 2 06/30 Write block 531 len 40960 data: 7 8 6 6 4 4 4 2 3 3 07/30 Flush 08/30 Write block 134 len 12288 data: 4 4 4 09/30 Read block 562 len 28672 10/30 Read block 251 len 4096 11/30 Flush 12/30 Write block 390 len 12288 data: 3 3 3 13/30 Write block 308 len 4096 data: 3 14/30 Flush 15/30 Write block 412 len 24576 data: 3 5 4 3 4 4 16/30 Read block 464 len 40960 17/30 Write block 501 len 4096 data: 5 18/30 Write block 155 len 40960 data: 3 3 3 3 3 3 4 4 3 3 19/30 Read block 546 len 36864 20/30 Write block 456 len 32768 data: 3 3 3 3 4 4 4 4 21/30 Write block 41 len 40960 data: 4 3 5 4 4 5 4 4 4 6 22/30 Write block 207 len 28672 data: 3 3 3 3 3 3 3 23/30 Read block 484 len 12288 24/30 Write block 151 len 16384 data: 3 3 4 3 25/30 Read block 496 len 8192 26/30 Read block 399 len 16384 27/30 Read block 551 len 20480 28/30 Read block 356 len 8192 29/30 Read block 38 len 32768 30/30 Write block 89 len 40960 data: 5 4 6 5 5 5 5 3 3 3 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 6 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:38.452492645Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1291} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 39 39 39 F F F 1 020-039 26 26 26 39 39 39 F F F 2 040-059 20 20 20 31 31 31 T T T 3 060-079 20 20 20 30 30 30 F F F 4 080-099 24 24 24 34 34 34 T T T 5 100-119 24 24 24 34 34 34 F F F 6 120-139 26 26 26 40 40 40 F F F 7 140-159 15 15 15 23 23 23 T T T 8 160-179 24 24 24 35 35 35 T T T 9 180-199 22 22 22 33 33 33 F F F 10 200-219 24 24 24 36 36 36 T T T 11 220-239 26 26 26 38 38 38 F F F 12 240-259 24 24 24 36 36 36 F F F 13 260-279 24 24 24 34 34 34 F F F 14 280-299 24 24 24 34 34 34 F F F 15 300-319 26 26 26 41 41 41 F F F 16 320-339 18 18 18 27 27 27 F F F 17 340-359 26 26 26 39 39 39 F F F 18 360-379 26 26 26 39 39 39 F F F 19 380-399 26 26 26 41 41 41 F F F 20 400-419 25 25 25 37 37 37 T T T 21 420-439 22 22 22 33 33 33 F F F 22 440-459 22 22 22 33 33 33 T T T 23 460-479 18 18 18 27 27 27 T T T 24 480-499 25 25 25 37 37 37 F F F 25 500-519 22 22 22 33 33 33 T T T 26 520-539 26 26 26 39 39 39 F F F 27 540-559 26 26 26 39 39 39 F F F 28 560-579 14 14 14 21 21 21 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 26, Max flush: 41 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 39 39 39 F F F 1 020-039 26 26 26 39 39 39 F F F 2 040-059 27 27 27 42 42 42 F F F 3 060-079 20 20 20 30 30 30 F F F 4 080-099 27 27 27 42 42 42 F F F 5 100-119 24 24 24 34 34 34 F F F 6 120-139 26 26 26 40 40 40 F F F 7 140-159 27 27 27 42 42 42 F F F 8 160-179 27 27 27 42 42 42 F F F 9 180-199 22 22 22 33 33 33 F F F 10 200-219 27 27 27 42 42 42 F F F 11 220-239 26 26 26 38 38 38 F F F 12 240-259 24 24 24 36 36 36 F F F 13 260-279 24 24 24 34 34 34 F F F 14 280-299 24 24 24 34 34 34 F F F 15 300-319 26 26 26 41 41 41 F F F 16 320-339 18 18 18 27 27 27 F F F 17 340-359 26 26 26 39 39 39 F F F 18 360-379 26 26 26 39 39 39 F F F 19 380-399 26 26 26 41 41 41 F F F 20 400-419 27 27 27 42 42 42 F F F 21 420-439 22 22 22 33 33 33 F F F 22 440-459 27 27 27 42 42 42 F F F 23 460-479 27 27 27 42 42 42 F F F 24 480-499 25 25 25 37 37 37 F F F 25 500-519 27 27 27 42 42 42 F F F 26 520-539 26 26 26 39 39 39 F F F 27 540-559 26 26 26 39 39 39 F F F 28 560-579 14 14 14 21 21 21 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 27, Max flush: 42 Begin loop 14 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.751601887Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.795468518Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.796450584Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.796495639Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.796511457Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.796526785Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299} {"msg":"Crucible 6dfd2dcd-2992-475c-ad58-435675a807a0 has session id: e3772466-9239-4406-bb94-f25abe41c8ec","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.797037875Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Upstairs opts: Upstairs UUID: 6dfd2dcd-2992-475c-ad58-435675a807a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.797095189Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Crucible stats registered with UUID: 6dfd2dcd-2992-475c-ad58-435675a807a0","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.797120386Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.827811524Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.827863438Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299} {"msg":"6dfd2dcd-2992-475c-ad58-435675a807a0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.827880386Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.827896624Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"0","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.82793058Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"0","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.827949368Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"1","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.827967166Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"1","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.827996513Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"2","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.828031139Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"2","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.828058845Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.828091502Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.828126688Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:56180)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.828216327Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:44547)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.828260632Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:63049)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.828304037Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.829358124Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"1","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.829412918Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"0","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.829508456Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"2","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830096188Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"1","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830126664Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"0","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830143032Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"2","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830626676Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830754681Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"[0]R flush_numbers[0..12]: [39, 39, 42, 30, 42, 34, 40, 42, 42, 33, 42, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830841911Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"[0]R generation[0..12]: [26, 26, 27, 20, 27, 24, 26, 27, 27, 22, 27, 26]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830861829Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830876527Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"[1]R flush_numbers[0..12]: [39, 39, 42, 30, 42, 34, 40, 42, 42, 33, 42, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830890475Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"[1]R generation[0..12]: [26, 26, 27, 20, 27, 24, 26, 27, 27, 22, 27, 26]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830904404Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830918452Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"[2]R flush_numbers[0..12]: [39, 39, 42, 30, 42, 34, 40, 42, 42, 33, 42, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.83093198Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"[2]R generation[0..12]: [26, 26, 27, 20, 27, 24, 26, 27, 27, 22, 27, 26]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830945669Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830959407Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830976145Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.830989854Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.831003102Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.831016151Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"":"downstairs","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"No downstairs reconciliation required","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.831037468Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.831049747Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"0","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.831063415Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"1","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.831076563Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"client":"2","":"downstairs","session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.831090222Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.83110264Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"6dfd2dcd-2992-475c-ad58-435675a807a0 is now active with session: e3772466-9239-4406-bb94-f25abe41c8ec","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.831115649Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.831127838Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299,"session_id":"e3772466-9239-4406-bb94-f25abe41c8ec"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:42.831140176Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 389 len 28672 data: 4 4 4 4 3 3 3 02/30 Write block 205 len 8192 data: 2 2 03/30 Read block 153 len 24576 04/30 Read block 532 len 28672 05/30 Write block 528 len 4096 data: 5 06/30 Write block 56 len 4096 data: 3 07/30 Write block 67 len 4096 data: 3 08/30 Read block 577 len 20480 09/30 Write block 569 len 8192 data: 3 4 10/30 Read block 302 len 4096 11/30 Write block 177 len 20480 data: 3 3 3 3 2 12/30 Write block 382 len 20480 data: 3 4 4 4 4 13/30 Write block 569 len 4096 data: 4 14/30 Flush 15/30 Write block 36 len 16384 data: 2 2 2 2 16/30 Write block 484 len 36864 data: 2 4 4 4 4 4 4 4 3 17/30 Write block 154 len 36864 data: 4 4 4 4 4 4 4 5 5 18/30 Read block 80 len 40960 19/30 Write block 481 len 36864 data: 2 2 2 3 5 5 5 5 5 20/30 Read block 288 len 4096 21/30 Write block 478 len 40960 data: 5 4 3 3 3 3 4 6 6 6 22/30 Read block 257 len 12288 23/30 Write block 377 len 4096 data: 2 24/30 Read block 230 len 40960 25/30 Write block 85 len 32768 data: 5 6 6 5 6 5 7 6 26/30 Read block 320 len 32768 27/30 Read block 325 len 36864 28/30 Read block 466 len 40960 29/30 Write block 397 len 40960 data: 3 2 2 3 3 3 3 3 4 4 30/30 Write block 135 len 12288 data: 5 5 4 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 4 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:42.849255252Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1299} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 39 39 39 F F F 1 020-039 26 26 26 39 39 39 T T T 2 040-059 28 28 28 43 43 43 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 27 27 27 42 42 42 T T T 5 100-119 24 24 24 34 34 34 F F F 6 120-139 26 26 26 40 40 40 T T T 7 140-159 27 27 27 42 42 42 T T T 8 160-179 28 28 28 43 43 43 T T T 9 180-199 28 28 28 43 43 43 F F F 10 200-219 28 28 28 43 43 43 F F F 11 220-239 26 26 26 38 38 38 F F F 12 240-259 24 24 24 36 36 36 F F F 13 260-279 24 24 24 34 34 34 F F F 14 280-299 24 24 24 34 34 34 F F F 15 300-319 26 26 26 41 41 41 F F F 16 320-339 18 18 18 27 27 27 F F F 17 340-359 26 26 26 39 39 39 F F F 18 360-379 26 26 26 39 39 39 T T T 19 380-399 28 28 28 43 43 43 T T T 20 400-419 27 27 27 42 42 42 T T T 21 420-439 22 22 22 33 33 33 F F F 22 440-459 27 27 27 42 42 42 F F F 23 460-479 27 27 27 42 42 42 T T T 24 480-499 25 25 25 37 37 37 T T T 25 500-519 27 27 27 42 42 42 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 26 26 26 39 39 39 F F F 28 560-579 28 28 28 43 43 43 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 28, Max flush: 43 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 39 39 39 F F F 1 020-039 29 29 29 44 44 44 F F F 2 040-059 28 28 28 43 43 43 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 29 29 29 44 44 44 F F F 5 100-119 24 24 24 34 34 34 F F F 6 120-139 29 29 29 44 44 44 F F F 7 140-159 29 29 29 44 44 44 F F F 8 160-179 29 29 29 44 44 44 F F F 9 180-199 28 28 28 43 43 43 F F F 10 200-219 28 28 28 43 43 43 F F F 11 220-239 26 26 26 38 38 38 F F F 12 240-259 24 24 24 36 36 36 F F F 13 260-279 24 24 24 34 34 34 F F F 14 280-299 24 24 24 34 34 34 F F F 15 300-319 26 26 26 41 41 41 F F F 16 320-339 18 18 18 27 27 27 F F F 17 340-359 26 26 26 39 39 39 F F F 18 360-379 29 29 29 44 44 44 F F F 19 380-399 29 29 29 44 44 44 F F F 20 400-419 29 29 29 44 44 44 F F F 21 420-439 22 22 22 33 33 33 F F F 22 440-459 27 27 27 42 42 42 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 29 29 29 44 44 44 F F F 25 500-519 27 27 27 42 42 42 F F F 26 520-539 28 28 28 43 43 43 F F F 27 540-559 26 26 26 39 39 39 F F F 28 560-579 28 28 28 43 43 43 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 29, Max flush: 44 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.157078097Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.202195462Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.203055502Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.203095427Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.203110045Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.203124394Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307} {"msg":"Crucible 3d4230b9-b6bc-4831-9640-5f75963e7a3a has session id: 7f5b3648-0cec-49b1-a84d-ad07c73cb271","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.203650662Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Upstairs opts: Upstairs UUID: 3d4230b9-b6bc-4831-9640-5f75963e7a3a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.203700796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Crucible stats registered with UUID: 3d4230b9-b6bc-4831-9640-5f75963e7a3a","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.203725454Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.234668643Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.234712768Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307} {"msg":"3d4230b9-b6bc-4831-9640-5f75963e7a3a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.234740875Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.234760912Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"0","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.234799797Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"0","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.234834453Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"1","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.23486928Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"1","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.234903676Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"2","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.234938891Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"2","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.234973037Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.235008713Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.23503251Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"ds_connection connected from Ok(127.0.0.1:35154)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.235296959Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"ds_connection connected from Ok(127.0.0.1:49214)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.235362672Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"ds_connection connected from Ok(127.0.0.1:52585)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.235398138Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.235797861Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"2","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.235845166Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"0","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.236125363Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"1","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.236601017Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"2","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.236642562Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"0","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.236674749Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"1","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237258981Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237285358Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"[0]R flush_numbers[0..12]: [39, 44, 43, 43, 44, 34, 44, 44, 44, 43, 43, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237312044Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"[0]R generation[0..12]: [26, 29, 28, 28, 29, 24, 29, 29, 29, 28, 28, 26]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237327303Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237342071Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"[1]R flush_numbers[0..12]: [39, 44, 43, 43, 44, 34, 44, 44, 44, 43, 43, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237355919Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"[1]R generation[0..12]: [26, 29, 28, 28, 29, 24, 29, 29, 29, 28, 28, 26]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237373417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237387775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"[2]R flush_numbers[0..12]: [39, 44, 43, 43, 44, 34, 44, 44, 44, 43, 43, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237401464Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"[2]R generation[0..12]: [26, 29, 28, 28, 29, 24, 29, 29, 29, 28, 28, 26]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237415242Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237429171Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237442299Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237455648Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237468786Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237481815Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"":"downstairs","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237495153Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237515711Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"0","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237528999Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"1","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237542068Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"client":"2","":"downstairs","session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237555406Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237567665Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"3d4230b9-b6bc-4831-9640-5f75963e7a3a is now active with session: 7f5b3648-0cec-49b1-a84d-ad07c73cb271","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237582263Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.237595651Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307,"session_id":"7f5b3648-0cec-49b1-a84d-ad07c73cb271"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:47.23760789Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 432 len 8192 data: 3 3 02/30 Flush 03/30 Write block 214 len 36864 data: 3 3 4 4 4 3 3 3 3 04/30 Write block 287 len 36864 data: 2 2 3 3 3 5 4 3 3 05/30 Write block 7 len 8192 data: 5 4 06/30 Write block 260 len 36864 data: 4 4 4 4 5 6 6 6 5 07/30 Read block 431 len 40960 08/30 Read block 492 len 32768 09/30 Read block 321 len 20480 10/30 Read block 136 len 12288 11/30 Write block 211 len 32768 data: 4 4 4 4 4 5 5 5 12/30 Flush 13/30 Write block 322 len 4096 data: 4 14/30 Read block 141 len 20480 15/30 Read block 50 len 8192 16/30 Write block 431 len 32768 data: 2 4 4 3 3 4 4 4 17/30 Read block 374 len 40960 18/30 Write block 156 len 20480 data: 5 5 5 5 5 19/30 Write block 429 len 4096 data: 2 20/30 Flush 21/30 Write block 533 len 24576 data: 7 7 5 5 5 3 22/30 Write block 241 len 28672 data: 4 3 2 2 2 2 2 23/30 Write block 198 len 36864 data: 2 2 2 2 2 2 2 3 3 24/30 Read block 94 len 20480 25/30 Write block 140 len 12288 data: 5 5 5 26/30 Read block 285 len 40960 27/30 Read block 25 len 4096 28/30 Write block 194 len 16384 data: 4 2 2 2 29/30 Write block 116 len 8192 data: 5 3 30/30 Write block 407 len 36864 data: 4 4 3 4 3 4 6 5 4 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 4 Done Done Sent false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 3 6 Done 9 8 7 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:47.256755906Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1307} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 46 46 46 F F F 1 020-039 29 29 29 44 44 44 F F F 2 040-059 28 28 28 43 43 43 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 29 29 29 44 44 44 F F F 5 100-119 24 24 24 34 34 34 T T T 6 120-139 29 29 29 44 44 44 F F F 7 140-159 30 30 30 47 47 47 T T T 8 160-179 30 30 30 47 47 47 F F F 9 180-199 28 28 28 43 43 43 T T T 10 200-219 30 30 30 46 46 46 T T T 11 220-239 30 30 30 46 46 46 F F F 12 240-259 24 24 24 36 36 36 T T T 13 260-279 30 30 30 46 46 46 F F F 14 280-299 30 30 30 46 46 46 F F F 15 300-319 26 26 26 41 41 41 F F F 16 320-339 30 30 30 47 47 47 F F F 17 340-359 26 26 26 39 39 39 F F F 18 360-379 29 29 29 44 44 44 F F F 19 380-399 29 29 29 44 44 44 F F F 20 400-419 29 29 29 44 44 44 T T T 21 420-439 30 30 30 47 47 47 F F F 22 440-459 27 27 27 42 42 42 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 29 29 29 44 44 44 F F F 25 500-519 27 27 27 42 42 42 F F F 26 520-539 28 28 28 43 43 43 T T T 27 540-559 26 26 26 39 39 39 F F F 28 560-579 28 28 28 43 43 43 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 30, Max flush: 47 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 46 46 46 F F F 1 020-039 29 29 29 44 44 44 F F F 2 040-059 28 28 28 43 43 43 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 29 29 29 44 44 44 F F F 5 100-119 31 31 31 48 48 48 F F F 6 120-139 29 29 29 44 44 44 F F F 7 140-159 31 31 31 48 48 48 F F F 8 160-179 30 30 30 47 47 47 F F F 9 180-199 31 31 31 48 48 48 F F F 10 200-219 31 31 31 48 48 48 F F F 11 220-239 30 30 30 46 46 46 F F F 12 240-259 31 31 31 48 48 48 F F F 13 260-279 30 30 30 46 46 46 F F F 14 280-299 30 30 30 46 46 46 F F F 15 300-319 26 26 26 41 41 41 F F F 16 320-339 30 30 30 47 47 47 F F F 17 340-359 26 26 26 39 39 39 F F F 18 360-379 29 29 29 44 44 44 F F F 19 380-399 29 29 29 44 44 44 F F F 20 400-419 31 31 31 48 48 48 F F F 21 420-439 30 30 30 47 47 47 F F F 22 440-459 27 27 27 42 42 42 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 29 29 29 44 44 44 F F F 25 500-519 27 27 27 42 42 42 F F F 26 520-539 31 31 31 48 48 48 F F F 27 540-559 26 26 26 39 39 39 F F F 28 560-579 28 28 28 43 43 43 F F F 29 580-599 22 22 22 33 33 33 F F F Max gen: 31, Max flush: 48 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.546183209Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.592141566Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.592998006Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.593054049Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.593082236Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.593106463Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315} {"msg":"Crucible 5ad363a5-543e-4c4f-88eb-a26b191ec8ee has session id: e55645b7-feae-4484-a66a-fc52f9d0e839","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.593599226Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Upstairs opts: Upstairs UUID: 5ad363a5-543e-4c4f-88eb-a26b191ec8ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.59364472Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Crucible stats registered with UUID: 5ad363a5-543e-4c4f-88eb-a26b191ec8ee","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.593669337Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.624772378Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.62484206Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315} {"msg":"5ad363a5-543e-4c4f-88eb-a26b191ec8ee active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.624863777Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.624884915Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"0","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.624933309Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"0","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.624969965Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"1","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.625004601Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"1","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.625034708Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"2","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.625067994Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"2","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.625108279Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.625142605Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.625166502Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"ds_connection connected from Ok(127.0.0.1:53699)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.62518467Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"ds_connection connected from Ok(127.0.0.1:37871)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.625436041Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"ds_connection connected from Ok(127.0.0.1:63938)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.625541838Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.626108882Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"0","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.626140948Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"2","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.626257735Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"1","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.626797342Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"2","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.626954224Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"1","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.62698422Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"0","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627318831Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627542825Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"[0]R flush_numbers[0..12]: [46, 44, 43, 43, 44, 48, 44, 48, 47, 48, 48, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627599868Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"[0]R generation[0..12]: [30, 29, 28, 28, 29, 31, 29, 31, 30, 31, 31, 30]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627620416Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627635464Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"[1]R flush_numbers[0..12]: [46, 44, 43, 43, 44, 48, 44, 48, 47, 48, 48, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627649192Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"[1]R generation[0..12]: [30, 29, 28, 28, 29, 31, 29, 31, 30, 31, 31, 30]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627663031Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627677169Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"[2]R flush_numbers[0..12]: [46, 44, 43, 43, 44, 48, 44, 48, 47, 48, 48, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627691078Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"[2]R generation[0..12]: [30, 29, 28, 28, 29, 31, 29, 31, 30, 31, 31, 30]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627704856Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627718744Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627732033Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627745191Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.62775827Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627771338Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"":"downstairs","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"No downstairs reconciliation required","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-01-13T15:52:51.627790706Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627805184Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"0","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627819093Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"1","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627832411Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"client":"2","":"downstairs","session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.62784588Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627858218Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"5ad363a5-543e-4c4f-88eb-a26b191ec8ee is now active with session: e55645b7-feae-4484-a66a-fc52f9d0e839","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627871157Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627883105Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315,"session_id":"e55645b7-feae-4484-a66a-fc52f9d0e839"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:51.627895234Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 499 len 28672 data: 4 6 6 5 5 4 5 02/30 Write block 157 len 16384 data: 6 6 6 6 03/30 Read block 102 len 24576 04/30 Write block 516 len 4096 data: 6 05/30 Write block 88 len 28672 data: 6 7 6 8 7 6 6 06/30 Flush 07/30 Write block 41 len 12288 data: 5 4 6 08/30 Flush 09/30 Write block 216 len 24576 data: 6 6 6 4 4 4 10/30 Read block 121 len 20480 11/30 Write block 572 len 20480 data: 3 2 2 3 3 12/30 Read block 427 len 24576 13/30 Write block 198 len 36864 data: 3 3 3 3 3 3 3 4 4 14/30 Flush 15/30 Write block 180 len 32768 data: 4 3 2 3 4 4 6 5 16/30 Read block 260 len 12288 17/30 Read block 74 len 4096 18/30 Read block 312 len 20480 19/30 Read block 28 len 28672 20/30 Read block 337 len 28672 21/30 Flush 22/30 Write block 302 len 8192 data: 3 3 23/30 Read block 481 len 8192 24/30 Read block 291 len 16384 25/30 Write block 323 len 4096 data: 4 26/30 Read block 130 len 24576 27/30 Read block 92 len 28672 28/30 Write block 509 len 20480 data: 3 3 3 3 2 29/30 Read block 121 len 8192 30/30 Write block 591 len 24576 data: 4 4 4 4 3 3 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:51.647429515Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1315} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 46 46 46 F F F 1 020-039 29 29 29 44 44 44 F F F 2 040-059 32 32 32 50 50 50 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 32 32 32 49 49 49 F F F 5 100-119 31 31 31 48 48 48 F F F 6 120-139 29 29 29 44 44 44 F F F 7 140-159 32 32 32 49 49 49 F F F 8 160-179 32 32 32 49 49 49 F F F 9 180-199 32 32 32 52 52 52 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 31 31 31 48 48 48 F F F 13 260-279 30 30 30 46 46 46 F F F 14 280-299 30 30 30 46 46 46 F F F 15 300-319 26 26 26 41 41 41 T T T 16 320-339 30 30 30 47 47 47 T T T 17 340-359 26 26 26 39 39 39 F F F 18 360-379 29 29 29 44 44 44 F F F 19 380-399 29 29 29 44 44 44 F F F 20 400-419 31 31 31 48 48 48 F F F 21 420-439 30 30 30 47 47 47 F F F 22 440-459 27 27 27 42 42 42 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 32 32 32 49 49 49 F F F 25 500-519 32 32 32 49 49 49 T T T 26 520-539 31 31 31 48 48 48 F F F 27 540-559 26 26 26 39 39 39 F F F 28 560-579 32 32 32 51 51 51 F F F 29 580-599 22 22 22 33 33 33 T T T Max gen: 32, Max flush: 52 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 46 46 46 F F F 1 020-039 29 29 29 44 44 44 F F F 2 040-059 32 32 32 50 50 50 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 32 32 32 49 49 49 F F F 5 100-119 31 31 31 48 48 48 F F F 6 120-139 29 29 29 44 44 44 F F F 7 140-159 32 32 32 49 49 49 F F F 8 160-179 32 32 32 49 49 49 F F F 9 180-199 32 32 32 52 52 52 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 31 31 31 48 48 48 F F F 13 260-279 30 30 30 46 46 46 F F F 14 280-299 30 30 30 46 46 46 F F F 15 300-319 33 33 33 53 53 53 F F F 16 320-339 33 33 33 53 53 53 F F F 17 340-359 26 26 26 39 39 39 F F F 18 360-379 29 29 29 44 44 44 F F F 19 380-399 29 29 29 44 44 44 F F F 20 400-419 31 31 31 48 48 48 F F F 21 420-439 30 30 30 47 47 47 F F F 22 440-459 27 27 27 42 42 42 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 32 32 32 49 49 49 F F F 25 500-519 33 33 33 53 53 53 F F F 26 520-539 31 31 31 48 48 48 F F F 27 540-559 26 26 26 39 39 39 F F F 28 560-579 32 32 32 51 51 51 F F F 29 580-599 33 33 33 53 53 53 F F F Max gen: 33, Max flush: 53 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:55.92558816Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:55.970134822Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:55.970884245Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:55.970936329Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:55.970952607Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:55.970968855Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323} {"msg":"Crucible 545ed1cf-aa2b-4656-a717-459a46da35f1 has session id: d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:55.971422882Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"Upstairs opts: Upstairs UUID: 545ed1cf-aa2b-4656-a717-459a46da35f1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:55.971461988Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"Crucible stats registered with UUID: 545ed1cf-aa2b-4656-a717-459a46da35f1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:55.971478676Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.001682401Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.001731686Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323} {"msg":"545ed1cf-aa2b-4656-a717-459a46da35f1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.001760482Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.0017802Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"0","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.001824535Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"0","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.0018665Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"1","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.001903486Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"1","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.001924513Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"2","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.001957939Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"2","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.001989956Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.002024511Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.002058717Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"ds_connection connected from Ok(127.0.0.1:49167)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.002091564Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38851)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.002150427Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"ds_connection connected from Ok(127.0.0.1:52358)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.00229369Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.002987399Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"2","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.003032054Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"0","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.00306835Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"1","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.00374825Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"1","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.003778907Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"2","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.003799294Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"0","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004397275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004425211Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"[0]R flush_numbers[0..12]: [46, 44, 50, 43, 49, 48, 44, 49, 49, 52, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004488574Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"[0]R generation[0..12]: [30, 29, 32, 28, 32, 31, 29, 32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004509531Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.00452453Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"[1]R flush_numbers[0..12]: [46, 44, 50, 43, 49, 48, 44, 49, 49, 52, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004538728Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"[1]R generation[0..12]: [30, 29, 32, 28, 32, 31, 29, 32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004552386Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004566495Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"[2]R flush_numbers[0..12]: [46, 44, 50, 43, 49, 48, 44, 49, 49, 52, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004580383Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"[2]R generation[0..12]: [30, 29, 32, 28, 32, 31, 29, 32, 32, 32, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004594061Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.00460806Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004621488Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004634827Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004658304Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004673042Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"":"downstairs","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004686531Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004698619Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"0","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004711858Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"1","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004725016Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"client":"2","":"downstairs","session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004738255Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004750593Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"545ed1cf-aa2b-4656-a717-459a46da35f1 is now active with session: d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004764552Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.00477688Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323,"session_id":"d9b1f250-1a72-4b40-805b-f9c0cd1ecfc9"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:52:56.004789239Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 44 len 24576 data: 5 5 6 5 5 5 02/30 Read block 441 len 4096 03/30 Read block 590 len 28672 04/30 Read block 2 len 8192 05/30 Read block 169 len 8192 06/30 Read block 236 len 4096 07/30 Read block 29 len 12288 08/30 Read block 191 len 40960 09/30 Flush 10/30 Write block 581 len 36864 data: 5 4 5 4 4 3 4 3 3 11/30 Read block 118 len 12288 12/30 Read block 24 len 4096 13/30 Read block 281 len 28672 14/30 Read block 244 len 28672 15/30 Read block 527 len 28672 16/30 Read block 210 len 12288 17/30 Flush 18/30 Write block 22 len 16384 data: 3 2 2 2 19/30 Read block 431 len 40960 20/30 Read block 197 len 4096 21/30 Flush 22/30 Write block 4 len 24576 data: 2 3 4 6 5 4 23/30 Read block 394 len 16384 24/30 Read block 109 len 12288 25/30 Write block 177 len 36864 data: 4 4 4 5 4 3 4 5 5 26/30 Flush 27/30 Write block 572 len 16384 data: 4 3 3 4 28/30 Write block 374 len 36864 data: 2 2 2 3 2 2 3 4 4 29/30 Write block 26 len 28672 data: 3 3 3 3 3 4 4 30/30 Write block 572 len 28672 data: 5 4 4 5 4 4 3 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:52:56.024413519Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1323} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 34 34 34 56 56 56 T T T 2 040-059 34 34 34 54 54 54 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 32 32 32 49 49 49 F F F 5 100-119 31 31 31 48 48 48 F F F 6 120-139 29 29 29 44 44 44 F F F 7 140-159 32 32 32 49 49 49 F F F 8 160-179 34 34 34 57 57 57 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 31 31 31 48 48 48 F F F 13 260-279 30 30 30 46 46 46 F F F 14 280-299 30 30 30 46 46 46 F F F 15 300-319 33 33 33 53 53 53 F F F 16 320-339 33 33 33 53 53 53 F F F 17 340-359 26 26 26 39 39 39 F F F 18 360-379 29 29 29 44 44 44 T T T 19 380-399 29 29 29 44 44 44 T T T 20 400-419 31 31 31 48 48 48 F F F 21 420-439 30 30 30 47 47 47 F F F 22 440-459 27 27 27 42 42 42 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 32 32 32 49 49 49 F F F 25 500-519 33 33 33 53 53 53 F F F 26 520-539 31 31 31 48 48 48 F F F 27 540-559 26 26 26 39 39 39 F F F 28 560-579 32 32 32 51 51 51 T T T 29 580-599 34 34 34 55 55 55 F F F Max gen: 34, Max flush: 57 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 34 34 34 54 54 54 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 32 32 32 49 49 49 F F F 5 100-119 31 31 31 48 48 48 F F F 6 120-139 29 29 29 44 44 44 F F F 7 140-159 32 32 32 49 49 49 F F F 8 160-179 34 34 34 57 57 57 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 32 32 32 51 51 51 F F F 12 240-259 31 31 31 48 48 48 F F F 13 260-279 30 30 30 46 46 46 F F F 14 280-299 30 30 30 46 46 46 F F F 15 300-319 33 33 33 53 53 53 F F F 16 320-339 33 33 33 53 53 53 F F F 17 340-359 26 26 26 39 39 39 F F F 18 360-379 35 35 35 58 58 58 F F F 19 380-399 35 35 35 58 58 58 F F F 20 400-419 31 31 31 48 48 48 F F F 21 420-439 30 30 30 47 47 47 F F F 22 440-459 27 27 27 42 42 42 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 32 32 32 49 49 49 F F F 25 500-519 33 33 33 53 53 53 F F F 26 520-539 31 31 31 48 48 48 F F F 27 540-559 26 26 26 39 39 39 F F F 28 560-579 35 35 35 58 58 58 F F F 29 580-599 34 34 34 55 55 55 F F F Max gen: 35, Max flush: 58 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.293222677Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.33733181Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.338067735Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.33810632Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.338121058Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.338135797Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331} {"msg":"Crucible 401277a6-0b5e-4963-a0aa-e747118aa481 has session id: 7a20cbdf-9397-441a-b7fd-6eba6aa77cf1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.338713829Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Upstairs opts: Upstairs UUID: 401277a6-0b5e-4963-a0aa-e747118aa481, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.338753924Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Crucible stats registered with UUID: 401277a6-0b5e-4963-a0aa-e747118aa481","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.338770193Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369527664Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369566709Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331} {"msg":"401277a6-0b5e-4963-a0aa-e747118aa481 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369604355Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369638641Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"0","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369685225Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"0","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369769896Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"1","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369810291Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"1","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369837828Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"2","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369867554Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"2","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369892241Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369917238Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369943745Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51776)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.369971992Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42876)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.370001498Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37630)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.370043884Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.370679449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"0","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.370722144Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"1","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.370914062Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"2","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.371471827Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"1","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.371507573Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"0","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.37152862Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"2","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372039991Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372067907Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"[0]R flush_numbers[0..12]: [57, 58, 54, 43, 49, 48, 44, 49, 57, 57, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.37213239Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"[0]R generation[0..12]: [34, 35, 34, 28, 32, 31, 29, 32, 34, 34, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372153617Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372168816Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"[1]R flush_numbers[0..12]: [57, 58, 54, 43, 49, 48, 44, 49, 57, 57, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372183084Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"[1]R generation[0..12]: [34, 35, 34, 28, 32, 31, 29, 32, 34, 34, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372196872Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372211161Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"[2]R flush_numbers[0..12]: [57, 58, 54, 43, 49, 48, 44, 49, 57, 57, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372224729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"[2]R generation[0..12]: [34, 35, 34, 28, 32, 31, 29, 32, 34, 34, 32, 32]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372238597Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372252496Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372278953Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372293771Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.37230698Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372320298Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"":"downstairs","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:53:00.372333696Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372354434Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"0","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372367952Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"1","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372381291Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"client":"2","":"downstairs","session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372394639Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372407008Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"401277a6-0b5e-4963-a0aa-e747118aa481 is now active with session: 7a20cbdf-9397-441a-b7fd-6eba6aa77cf1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372419906Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372431985Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331,"session_id":"7a20cbdf-9397-441a-b7fd-6eba6aa77cf1"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:00.372444373Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 342 len 36864 data: 3 3 3 3 3 3 3 3 4 02/30 Read block 427 len 24576 03/30 Write block 235 len 16384 data: 2 2 2 3 04/30 Write block 87 len 28672 data: 7 7 8 7 9 8 7 05/30 Write block 288 len 4096 data: 3 06/30 Write block 348 len 4096 data: 4 07/30 Flush 08/30 Write block 61 len 32768 data: 2 2 3 4 3 3 4 5 09/30 Write block 397 len 40960 data: 4 3 3 4 4 4 4 4 5 5 10/30 Read block 327 len 12288 11/30 Read block 537 len 36864 12/30 Write block 166 len 24576 data: 4 4 5 5 5 3 13/30 Read block 348 len 32768 14/30 Write block 423 len 12288 data: 3 3 2 15/30 Read block 64 len 24576 16/30 Write block 313 len 24576 data: 2 3 3 4 3 3 17/30 Read block 179 len 8192 18/30 Read block 280 len 8192 19/30 Read block 578 len 4096 20/30 Write block 113 len 16384 data: 2 3 4 6 21/30 Write block 166 len 36864 data: 5 5 6 6 6 4 3 2 2 22/30 Flush 23/30 Write block 241 len 40960 data: 5 4 3 3 3 3 3 4 4 4 24/30 Write block 212 len 16384 data: 5 5 5 5 25/30 Read block 26 len 12288 26/30 Read block 250 len 4096 27/30 Write block 258 len 40960 data: 3 4 5 5 5 5 6 7 7 7 28/30 Write block 82 len 24576 data: 4 4 4 6 7 8 29/30 Read block 359 len 12288 30/30 Write block 242 len 28672 data: 5 4 4 4 4 4 5 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:00.392496573Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1331} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 34 34 34 54 54 54 F F F 3 060-079 36 36 36 60 60 60 F F F 4 080-099 36 36 36 59 59 59 T T T 5 100-119 36 36 36 60 60 60 F F F 6 120-139 29 29 29 44 44 44 F F F 7 140-159 32 32 32 49 49 49 F F F 8 160-179 36 36 36 60 60 60 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 32 32 32 51 51 51 T T T 11 220-239 36 36 36 59 59 59 F F F 12 240-259 31 31 31 48 48 48 T T T 13 260-279 30 30 30 46 46 46 T T T 14 280-299 36 36 36 59 59 59 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 33 33 33 53 53 53 F F F 17 340-359 36 36 36 59 59 59 F F F 18 360-379 35 35 35 58 58 58 F F F 19 380-399 36 36 36 60 60 60 F F F 20 400-419 36 36 36 60 60 60 F F F 21 420-439 36 36 36 60 60 60 F F F 22 440-459 27 27 27 42 42 42 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 32 32 32 49 49 49 F F F 25 500-519 33 33 33 53 53 53 F F F 26 520-539 31 31 31 48 48 48 F F F 27 540-559 26 26 26 39 39 39 F F F 28 560-579 35 35 35 58 58 58 F F F 29 580-599 34 34 34 55 55 55 F F F Max gen: 36, Max flush: 60 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 34 34 34 54 54 54 F F F 3 060-079 36 36 36 60 60 60 F F F 4 080-099 37 37 37 61 61 61 F F F 5 100-119 36 36 36 60 60 60 F F F 6 120-139 29 29 29 44 44 44 F F F 7 140-159 32 32 32 49 49 49 F F F 8 160-179 36 36 36 60 60 60 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 37 37 37 61 61 61 F F F 11 220-239 36 36 36 59 59 59 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 37 37 37 61 61 61 F F F 14 280-299 36 36 36 59 59 59 F F F 15 300-319 36 36 36 60 60 60 F F F 16 320-339 33 33 33 53 53 53 F F F 17 340-359 36 36 36 59 59 59 F F F 18 360-379 35 35 35 58 58 58 F F F 19 380-399 36 36 36 60 60 60 F F F 20 400-419 36 36 36 60 60 60 F F F 21 420-439 36 36 36 60 60 60 F F F 22 440-459 27 27 27 42 42 42 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 32 32 32 49 49 49 F F F 25 500-519 33 33 33 53 53 53 F F F 26 520-539 31 31 31 48 48 48 F F F 27 540-559 26 26 26 39 39 39 F F F 28 560-579 35 35 35 58 58 58 F F F 29 580-599 34 34 34 55 55 55 F F F Max gen: 37, Max flush: 61 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.663654427Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.707988654Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.708839825Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.708887299Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.708904657Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.708922355Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339} {"msg":"Crucible 9d54979f-4766-4bf6-bd23-c0eaa949bc53 has session id: 2451b182-e81a-4001-a876-4bc0d2658d61","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.709404449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Upstairs opts: Upstairs UUID: 9d54979f-4766-4bf6-bd23-c0eaa949bc53, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.709443194Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Crucible stats registered with UUID: 9d54979f-4766-4bf6-bd23-c0eaa949bc53","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.709464622Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.739461881Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.739534413Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339} {"msg":"9d54979f-4766-4bf6-bd23-c0eaa949bc53 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.73955823Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.739576598Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"0","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.739623173Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"0","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.739659398Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"1","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.739695154Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"1","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.739726911Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"2","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.739762246Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"2","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.739795083Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.739827849Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.739857785Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"ds_connection connected from Ok(127.0.0.1:46154)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.739889632Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"ds_connection connected from Ok(127.0.0.1:46084)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.740096958Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"ds_connection connected from Ok(127.0.0.1:53145)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.740141112Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.740654002Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"0","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.740684679Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"1","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.74076367Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"2","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.741289758Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"1","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.741365739Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"2","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.741391666Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"0","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.741955521Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742023413Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"[0]R flush_numbers[0..12]: [57, 58, 54, 60, 61, 60, 44, 49, 60, 57, 61, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742052699Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"[0]R generation[0..12]: [34, 35, 34, 36, 37, 36, 29, 32, 36, 34, 37, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742078346Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742104983Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"[1]R flush_numbers[0..12]: [57, 58, 54, 60, 61, 60, 44, 49, 60, 57, 61, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.74213039Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"[1]R generation[0..12]: [34, 35, 34, 36, 37, 36, 29, 32, 36, 34, 37, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742153838Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742182964Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"[2]R flush_numbers[0..12]: [57, 58, 54, 60, 61, 60, 44, 49, 60, 57, 61, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742211301Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"[2]R generation[0..12]: [34, 35, 34, 36, 37, 36, 29, 32, 36, 34, 37, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742238477Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742265194Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:53:04.742293841Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742322618Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742350395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742379101Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"":"downstairs","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742408318Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742434475Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"0","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742463481Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"1","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742491408Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"client":"2","":"downstairs","session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742523014Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742544762Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"9d54979f-4766-4bf6-bd23-c0eaa949bc53 is now active with session: 2451b182-e81a-4001-a876-4bc0d2658d61","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.7425593Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742574708Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339,"session_id":"2451b182-e81a-4001-a876-4bc0d2658d61"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:04.742587417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 388 len 28672 data: 4 5 5 5 5 4 4 02/30 Read block 130 len 40960 03/30 Write block 337 len 40960 data: 3 3 3 2 3 4 4 4 4 4 04/30 Read block 472 len 20480 05/30 Flush 06/30 Write block 325 len 20480 data: 3 2 2 3 3 07/30 Read block 3 len 24576 08/30 Write block 371 len 12288 data: 2 2 2 09/30 Read block 418 len 36864 10/30 Write block 174 len 16384 data: 3 2 3 5 11/30 Read block 271 len 28672 12/30 Read block 475 len 36864 13/30 Write block 444 len 12288 data: 3 3 3 14/30 Read block 145 len 8192 15/30 Flush 16/30 Write block 136 len 40960 data: 6 5 4 4 6 6 6 4 4 3 17/30 Read block 355 len 32768 18/30 Read block 166 len 12288 19/30 Write block 545 len 36864 data: 3 4 4 5 5 5 6 6 6 20/30 Read block 169 len 12288 21/30 Read block 534 len 32768 22/30 Write block 544 len 12288 data: 3 4 5 23/30 Flush 24/30 Write block 297 len 36864 data: 3 3 3 3 3 4 4 3 3 25/30 Read block 471 len 8192 26/30 Read block 474 len 16384 27/30 Read block 513 len 40960 28/30 Write block 543 len 20480 data: 3 4 5 6 5 29/30 Read block 234 len 8192 30/30 Read block 149 len 36864 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:04.76424269Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1339} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 34 34 34 54 54 54 F F F 3 060-079 36 36 36 60 60 60 F F F 4 080-099 37 37 37 61 61 61 F F F 5 100-119 36 36 36 60 60 60 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 38 38 38 63 63 63 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 37 37 37 61 61 61 F F F 11 220-239 36 36 36 59 59 59 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 37 37 37 61 61 61 F F F 14 280-299 36 36 36 59 59 59 T T T 15 300-319 36 36 36 60 60 60 T T T 16 320-339 38 38 38 63 63 63 F F F 17 340-359 38 38 38 62 62 62 F F F 18 360-379 38 38 38 63 63 63 F F F 19 380-399 38 38 38 62 62 62 F F F 20 400-419 36 36 36 60 60 60 F F F 21 420-439 36 36 36 60 60 60 F F F 22 440-459 38 38 38 63 63 63 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 32 32 32 49 49 49 F F F 25 500-519 33 33 33 53 53 53 F F F 26 520-539 31 31 31 48 48 48 F F F 27 540-559 38 38 38 64 64 64 T T T 28 560-579 35 35 35 58 58 58 F F F 29 580-599 34 34 34 55 55 55 F F F Max gen: 38, Max flush: 64 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 34 34 34 54 54 54 F F F 3 060-079 36 36 36 60 60 60 F F F 4 080-099 37 37 37 61 61 61 F F F 5 100-119 36 36 36 60 60 60 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 38 38 38 63 63 63 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 37 37 37 61 61 61 F F F 11 220-239 36 36 36 59 59 59 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 37 37 37 61 61 61 F F F 14 280-299 39 39 39 65 65 65 F F F 15 300-319 39 39 39 65 65 65 F F F 16 320-339 38 38 38 63 63 63 F F F 17 340-359 38 38 38 62 62 62 F F F 18 360-379 38 38 38 63 63 63 F F F 19 380-399 38 38 38 62 62 62 F F F 20 400-419 36 36 36 60 60 60 F F F 21 420-439 36 36 36 60 60 60 F F F 22 440-459 38 38 38 63 63 63 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 32 32 32 49 49 49 F F F 25 500-519 33 33 33 53 53 53 F F F 26 520-539 31 31 31 48 48 48 F F F 27 540-559 39 39 39 65 65 65 F F F 28 560-579 35 35 35 58 58 58 F F F 29 580-599 34 34 34 55 55 55 F F F Max gen: 39, Max flush: 65 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.02633138Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.069835904Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.070515644Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.07055355Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.070568248Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.070582917Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347} {"msg":"Crucible 93b01e22-7b3b-45e3-b5cd-2132e7a2cc63 has session id: 4361419d-ccad-4e7f-974a-e6af6a5779c0","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.071033644Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Upstairs opts: Upstairs UUID: 93b01e22-7b3b-45e3-b5cd-2132e7a2cc63, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.071078399Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Crucible stats registered with UUID: 93b01e22-7b3b-45e3-b5cd-2132e7a2cc63","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.071102826Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.100942384Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.100987449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347} {"msg":"93b01e22-7b3b-45e3-b5cd-2132e7a2cc63 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101019915Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101037793Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"0","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101067479Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"0","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101085967Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"1","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101103795Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"1","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101120683Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"2","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101138451Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"2","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101154879Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101172027Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101194845Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:63062)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101325399Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:43577)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101554403Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:35244)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.101597058Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.102152243Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"1","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.102232323Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"0","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.102364018Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"2","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.102913304Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"1","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.102956599Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"0","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.103180923Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"2","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.103498156Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.103521793Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"[0]R flush_numbers[0..12]: [57, 58, 54, 60, 61, 60, 64, 64, 63, 57, 61, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.103790162Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"[0]R generation[0..12]: [34, 35, 34, 36, 37, 36, 38, 38, 38, 34, 37, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.103820218Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.103842826Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"[1]R flush_numbers[0..12]: [57, 58, 54, 60, 61, 60, 64, 64, 63, 57, 61, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.103860544Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"[1]R generation[0..12]: [34, 35, 34, 36, 37, 36, 38, 38, 38, 34, 37, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.103877491Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.103895129Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"[2]R flush_numbers[0..12]: [57, 58, 54, 60, 61, 60, 64, 64, 63, 57, 61, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.103912547Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"[2]R generation[0..12]: [34, 35, 34, 36, 37, 36, 38, 38, 38, 34, 37, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.103930085Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104048302Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104086717Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104121813Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104154279Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104184606Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"":"downstairs","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104216812Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104243209Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"0","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104273765Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"1"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104312111Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"client":"2","":"downstairs","session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104341237Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104357815Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"93b01e22-7b3b-45e3-b5cd-2132e7a2cc63 is now active with session: 4361419d-ccad-4e7f-974a-e6af6a5779c0","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104374593Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.104390332Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347,"session_id":"4361419d-ccad-4e7f-974a-e6af6a5779c0"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:09.10440602Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 99 len 28672 data: 3 3 3 4 4 5 5 02/30 Read block 336 len 20480 03/30 Read block 368 len 24576 04/30 Write block 288 len 28672 data: 4 4 4 4 6 5 4 05/30 Read block 547 len 20480 06/30 Write block 274 len 12288 data: 4 4 4 07/30 Read block 268 len 16384 08/30 Write block 104 len 36864 data: 6 6 3 3 3 2 2 2 2 09/30 Read block 362 len 8192 10/30 Write block 421 len 24576 data: 4 4 4 4 3 2 11/30 Write block 44 len 12288 data: 6 6 7 12/30 Flush 13/30 Write block 578 len 32768 data: 4 3 5 6 5 6 5 5 14/30 Write block 309 len 4096 data: 2 15/30 Write block 213 len 8192 data: 6 6 16/30 Write block 389 len 20480 data: 6 6 6 6 5 17/30 Read block 562 len 12288 18/30 Write block 455 len 20480 data: 3 4 4 4 4 19/30 Read block 235 len 28672 20/30 Write block 408 len 20480 data: 5 4 5 4 5 21/30 Write block 589 len 40960 data: 4 3 5 5 5 5 4 4 3 3 22/30 Flush 23/30 Write block 449 len 40960 data: 4 4 4 4 4 3 4 5 5 5 24/30 Read block 346 len 32768 25/30 Write block 160 len 36864 data: 7 6 6 4 4 3 6 6 7 26/30 Write block 195 len 4096 data: 3 27/30 Read block 210 len 40960 28/30 Write block 320 len 28672 data: 4 4 5 5 3 4 3 29/30 Write block 363 len 24576 data: 5 4 4 4 4 3 30/30 Read block 258 len 40960 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:09.124405486Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1347} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 40 40 40 66 66 66 F F F 3 060-079 36 36 36 60 60 60 F F F 4 080-099 40 40 40 66 66 66 F F F 5 100-119 40 40 40 66 66 66 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 38 38 38 63 63 63 T T T 9 180-199 34 34 34 57 57 57 T T T 10 200-219 40 40 40 67 67 67 F F F 11 220-239 36 36 36 59 59 59 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 40 40 40 66 66 66 F F F 14 280-299 40 40 40 66 66 66 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 38 38 38 63 63 63 T T T 17 340-359 38 38 38 62 62 62 F F F 18 360-379 38 38 38 63 63 63 T T T 19 380-399 40 40 40 67 67 67 F F F 20 400-419 40 40 40 67 67 67 F F F 21 420-439 40 40 40 66 66 66 F F F 22 440-459 40 40 40 67 67 67 T T T 23 460-479 29 29 29 44 44 44 F F F 24 480-499 32 32 32 49 49 49 F F F 25 500-519 33 33 33 53 53 53 F F F 26 520-539 31 31 31 48 48 48 F F F 27 540-559 39 39 39 65 65 65 F F F 28 560-579 40 40 40 67 67 67 F F F 29 580-599 40 40 40 67 67 67 F F F Max gen: 40, Max flush: 67 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 40 40 40 66 66 66 F F F 3 060-079 36 36 36 60 60 60 F F F 4 080-099 40 40 40 66 66 66 F F F 5 100-119 40 40 40 66 66 66 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 41 41 41 68 68 68 F F F 9 180-199 41 41 41 68 68 68 F F F 10 200-219 40 40 40 67 67 67 F F F 11 220-239 36 36 36 59 59 59 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 40 40 40 66 66 66 F F F 14 280-299 40 40 40 66 66 66 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 41 41 41 68 68 68 F F F 17 340-359 38 38 38 62 62 62 F F F 18 360-379 41 41 41 68 68 68 F F F 19 380-399 40 40 40 67 67 67 F F F 20 400-419 40 40 40 67 67 67 F F F 21 420-439 40 40 40 66 66 66 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 32 32 32 49 49 49 F F F 25 500-519 33 33 33 53 53 53 F F F 26 520-539 31 31 31 48 48 48 F F F 27 540-559 39 39 39 65 65 65 F F F 28 560-579 40 40 40 67 67 67 F F F 29 580-599 40 40 40 67 67 67 F F F Max gen: 41, Max flush: 68 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.400284299Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.445907035Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.447109505Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.447149861Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.447172648Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.447197705Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355} {"msg":"Crucible fdb3cbaa-8e5f-4ff3-91b0-bd4e407d319d has session id: 099dae4d-f94a-4657-9750-baf801e5084b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.44767256Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Upstairs opts: Upstairs UUID: fdb3cbaa-8e5f-4ff3-91b0-bd4e407d319d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.447712355Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Crucible stats registered with UUID: fdb3cbaa-8e5f-4ff3-91b0-bd4e407d319d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.447729333Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.47852542Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478583093Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355} {"msg":"fdb3cbaa-8e5f-4ff3-91b0-bd4e407d319d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478601041Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478617129Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"0","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478646215Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"0","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478663833Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"1","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478681511Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"1","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478704419Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"2","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478728206Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"2","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478757133Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478790909Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478815326Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40963)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478911914Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"ds_connection connected from Ok(127.0.0.1:39485)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.478939311Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52920)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.479197341Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.479738248Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"0","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.479772104Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"1","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.480053451Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"2","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.480531405Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"0","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.480573281Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"1","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.480627024Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"2","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481085661Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.48117999Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"[0]R flush_numbers[0..12]: [57, 58, 66, 60, 66, 66, 64, 64, 68, 68, 67, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481257511Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"[0]R generation[0..12]: [34, 35, 40, 36, 40, 40, 38, 38, 41, 41, 40, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481277858Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481293006Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"[1]R flush_numbers[0..12]: [57, 58, 66, 60, 66, 66, 64, 64, 68, 68, 67, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481307355Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"[1]R generation[0..12]: [34, 35, 40, 36, 40, 40, 38, 38, 41, 41, 40, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481321513Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481335672Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"[2]R flush_numbers[0..12]: [57, 58, 66, 60, 66, 66, 64, 64, 68, 68, 67, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.48134935Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"[2]R generation[0..12]: [34, 35, 40, 36, 40, 40, 38, 38, 41, 41, 40, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481362948Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481377097Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481390045Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481403254Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481416122Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.48143378Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"":"downstairs","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"Wait for a query_work_queue command to finish before sending IO msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481508691Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481526169Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"0","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481539928Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"1","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481553166Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"client":"2","":"downstairs","session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481566475Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481578673Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"fdb3cbaa-8e5f-4ff3-91b0-bd4e407d319d is now active with session: 099dae4d-f94a-4657-9750-baf801e5084b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481591522Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.4816035Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355,"session_id":"099dae4d-f94a-4657-9750-baf801e5084b"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:13.481615779Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 66 len 8192 data: 4 5 02/30 Read block 589 len 40960 03/30 Write block 402 len 8192 data: 5 5 04/30 Read block 588 len 32768 05/30 Flush 06/30 Write block 571 len 32768 data: 3 6 5 5 6 5 5 5 07/30 Write block 103 len 4096 data: 5 08/30 Read block 206 len 20480 09/30 Write block 534 len 40960 data: 8 6 6 6 4 4 4 3 3 4 10/30 Read block 468 len 8192 11/30 Read block 101 len 16384 12/30 Read block 210 len 8192 13/30 Read block 371 len 40960 14/30 Read block 235 len 20480 15/30 Write block 506 len 40960 data: 6 5 5 4 4 4 4 3 4 4 16/30 Read block 575 len 40960 17/30 Read block 155 len 40960 18/30 Flush 19/30 Write block 479 len 8192 data: 5 4 20/30 Read block 462 len 8192 21/30 Write block 324 len 8192 data: 4 5 22/30 Write block 531 len 8192 data: 8 9 23/30 Read block 168 len 4096 24/30 Read block 317 len 8192 25/30 Read block 377 len 20480 26/30 Write block 466 len 16384 data: 2 3 3 3 27/30 Write block 95 len 20480 data: 6 4 4 4 4 28/30 Read block 513 len 28672 29/30 Read block 129 len 20480 30/30 Read block 413 len 20480 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:13.501277425Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1355} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 40 40 40 66 66 66 F F F 3 060-079 42 42 42 69 69 69 F F F 4 080-099 40 40 40 66 66 66 T T T 5 100-119 42 42 42 70 70 70 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 41 41 41 68 68 68 F F F 9 180-199 41 41 41 68 68 68 F F F 10 200-219 40 40 40 67 67 67 F F F 11 220-239 36 36 36 59 59 59 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 40 40 40 66 66 66 F F F 14 280-299 40 40 40 66 66 66 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 41 41 41 68 68 68 T T T 17 340-359 38 38 38 62 62 62 F F F 18 360-379 41 41 41 68 68 68 F F F 19 380-399 40 40 40 67 67 67 F F F 20 400-419 42 42 42 69 69 69 F F F 21 420-439 40 40 40 66 66 66 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 29 29 29 44 44 44 T T T 24 480-499 32 32 32 49 49 49 T T T 25 500-519 42 42 42 70 70 70 F F F 26 520-539 42 42 42 70 70 70 T T T 27 540-559 42 42 42 70 70 70 F F F 28 560-579 42 42 42 70 70 70 F F F 29 580-599 40 40 40 67 67 67 F F F Max gen: 42, Max flush: 70 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 40 40 40 66 66 66 F F F 3 060-079 42 42 42 69 69 69 F F F 4 080-099 43 43 43 71 71 71 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 41 41 41 68 68 68 F F F 9 180-199 41 41 41 68 68 68 F F F 10 200-219 40 40 40 67 67 67 F F F 11 220-239 36 36 36 59 59 59 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 40 40 40 66 66 66 F F F 14 280-299 40 40 40 66 66 66 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 43 43 43 71 71 71 F F F 17 340-359 38 38 38 62 62 62 F F F 18 360-379 41 41 41 68 68 68 F F F 19 380-399 40 40 40 67 67 67 F F F 20 400-419 42 42 42 69 69 69 F F F 21 420-439 40 40 40 66 66 66 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 43 43 43 71 71 71 F F F 24 480-499 43 43 43 71 71 71 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 43 43 43 71 71 71 F F F 27 540-559 42 42 42 70 70 70 F F F 28 560-579 42 42 42 70 70 70 F F F 29 580-599 40 40 40 67 67 67 F F F Max gen: 43, Max flush: 71 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.778917393Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.824804449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.825764717Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.825804852Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.82581981Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.825834328Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363} {"msg":"Crucible ab27afc9-c568-4c59-9baa-c5738a46823d has session id: a4deeea1-9b57-49e5-a95c-1f1ec913bc8b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.826271047Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Upstairs opts: Upstairs UUID: ab27afc9-c568-4c59-9baa-c5738a46823d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.826315012Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Crucible stats registered with UUID: ab27afc9-c568-4c59-9baa-c5738a46823d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.826339099Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.856832761Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.856892234Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363} {"msg":"ab27afc9-c568-4c59-9baa-c5738a46823d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.856909842Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.85693166Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"0","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.856962366Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"0","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.856990283Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"1","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.857024629Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"1","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.857060915Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"2","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.85709943Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"2","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.857133896Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.857167772Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.857200808Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48779)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.857254402Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33317)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.857293538Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:59392)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.857327123Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.858233148Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"0","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.858267914Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"1","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.858451292Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"2","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859025635Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"1","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859056432Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"2","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859078389Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"0","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859607717Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859634994Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"[0]R flush_numbers[0..12]: [57, 58, 66, 69, 71, 70, 64, 64, 68, 68, 67, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859691038Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"[0]R generation[0..12]: [34, 35, 40, 42, 43, 42, 38, 38, 41, 41, 40, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859710916Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859727023Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"[1]R flush_numbers[0..12]: [57, 58, 66, 69, 71, 70, 64, 64, 68, 68, 67, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859741772Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"[1]R generation[0..12]: [34, 35, 40, 42, 43, 42, 38, 38, 41, 41, 40, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.85975615Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859770438Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"[2]R flush_numbers[0..12]: [57, 58, 66, 69, 71, 70, 64, 64, 68, 68, 67, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859784167Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"[2]R generation[0..12]: [34, 35, 40, 42, 43, 42, 38, 38, 41, 41, 40, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859797825Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859815923Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859830311Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.85984384Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859857088Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859870847Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"":"downstairs","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:53:17.859884245Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859903853Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"0","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859917471Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"1","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.8599307Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"client":"2","":"downstairs","session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859944178Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859956407Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"ab27afc9-c568-4c59-9baa-c5738a46823d is now active with session: a4deeea1-9b57-49e5-a95c-1f1ec913bc8b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859969195Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.859982054Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363,"session_id":"a4deeea1-9b57-49e5-a95c-1f1ec913bc8b"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:17.86001148Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 533 len 36864 data: 8 9 7 7 7 5 5 5 4 02/30 Read block 511 len 16384 03/30 Read block 386 len 20480 04/30 Read block 253 len 24576 05/30 Write block 276 len 32768 data: 5 5 4 5 5 4 4 3 06/30 Read block 347 len 28672 07/30 Read block 227 len 28672 08/30 Read block 47 len 28672 09/30 Flush 10/30 Write block 354 len 24576 data: 4 5 5 5 5 6 11/30 Write block 78 len 4096 data: 2 12/30 Read block 480 len 20480 13/30 Read block 568 len 40960 14/30 Read block 538 len 28672 15/30 Read block 534 len 40960 16/30 Flush 17/30 Write block 161 len 20480 data: 7 7 5 5 4 18/30 Write block 397 len 12288 data: 5 4 4 19/30 Read block 146 len 20480 20/30 Read block 37 len 40960 21/30 Read block 222 len 4096 22/30 Read block 537 len 24576 23/30 Read block 397 len 12288 24/30 Write block 571 len 12288 data: 4 7 6 25/30 Read block 19 len 20480 26/30 Write block 0 len 36864 data: 2 2 2 2 3 4 5 7 6 27/30 Write block 371 len 20480 data: 3 3 3 3 3 28/30 Read block 470 len 20480 29/30 Read block 542 len 8192 30/30 Write block 71 len 16384 data: 5 4 3 3 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 5 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:17.881893727Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1363} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 57 57 57 T T T 1 020-039 35 35 35 58 58 58 F F F 2 040-059 40 40 40 66 66 66 F F F 3 060-079 44 44 44 73 73 73 T T T 4 080-099 43 43 43 71 71 71 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 41 41 41 68 68 68 T T T 9 180-199 41 41 41 68 68 68 F F F 10 200-219 40 40 40 67 67 67 F F F 11 220-239 36 36 36 59 59 59 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 44 44 44 72 72 72 F F F 14 280-299 44 44 44 72 72 72 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 43 43 43 71 71 71 F F F 17 340-359 44 44 44 73 73 73 F F F 18 360-379 41 41 41 68 68 68 T T T 19 380-399 40 40 40 67 67 67 T T T 20 400-419 42 42 42 69 69 69 F F F 21 420-439 40 40 40 66 66 66 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 43 43 43 71 71 71 F F F 24 480-499 43 43 43 71 71 71 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 44 44 44 72 72 72 F F F 28 560-579 42 42 42 70 70 70 T T T 29 580-599 40 40 40 67 67 67 F F F Max gen: 44, Max flush: 73 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 74 74 74 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 40 40 40 66 66 66 F F F 3 060-079 45 45 45 74 74 74 F F F 4 080-099 43 43 43 71 71 71 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 45 45 45 74 74 74 F F F 9 180-199 41 41 41 68 68 68 F F F 10 200-219 40 40 40 67 67 67 F F F 11 220-239 36 36 36 59 59 59 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 44 44 44 72 72 72 F F F 14 280-299 44 44 44 72 72 72 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 43 43 43 71 71 71 F F F 17 340-359 44 44 44 73 73 73 F F F 18 360-379 45 45 45 74 74 74 F F F 19 380-399 45 45 45 74 74 74 F F F 20 400-419 42 42 42 69 69 69 F F F 21 420-439 40 40 40 66 66 66 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 43 43 43 71 71 71 F F F 24 480-499 43 43 43 71 71 71 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 44 44 44 72 72 72 F F F 28 560-579 45 45 45 74 74 74 F F F 29 580-599 40 40 40 67 67 67 F F F Max gen: 45, Max flush: 74 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.170646178Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.214637704Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.215467907Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.215508253Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.215522941Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.215538209Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371} {"msg":"Crucible 96c7dacc-cc15-464f-8958-53883d4444f7 has session id: 2a67742a-687e-4f21-9853-e15987bcd258","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.215944702Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Upstairs opts: Upstairs UUID: 96c7dacc-cc15-464f-8958-53883d4444f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.215994226Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Crucible stats registered with UUID: 96c7dacc-cc15-464f-8958-53883d4444f7","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.216022363Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246349384Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246407077Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371} {"msg":"96c7dacc-cc15-464f-8958-53883d4444f7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246430395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246448063Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"0","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246490118Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"0","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246525294Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"1","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246561019Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"1","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246593126Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"2","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246627812Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"2","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246660458Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246679206Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246696344Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"ds_connection connected from Ok(127.0.0.1:45017)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246716451Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"ds_connection connected from Ok(127.0.0.1:55429)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246756916Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"ds_connection connected from Ok(127.0.0.1:54007)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.246804441Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.247440117Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"0","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.247574901Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"1","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.247879965Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"2","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.24809592Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"0","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.248263431Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"1","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.248617519Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"2","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.248775321Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.248945581Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"[0]R flush_numbers[0..12]: [74, 58, 66, 74, 71, 70, 64, 64, 74, 68, 67, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249158396Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"[0]R generation[0..12]: [45, 35, 40, 45, 43, 42, 38, 38, 45, 41, 40, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249181974Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249197382Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"[1]R flush_numbers[0..12]: [74, 58, 66, 74, 71, 70, 64, 64, 74, 68, 67, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.24921158Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"[1]R generation[0..12]: [45, 35, 40, 45, 43, 42, 38, 38, 45, 41, 40, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249225279Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249242597Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"[2]R flush_numbers[0..12]: [74, 58, 66, 74, 71, 70, 64, 64, 74, 68, 67, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249256555Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"[2]R generation[0..12]: [45, 35, 40, 45, 43, 42, 38, 38, 45, 41, 40, 36]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249271613Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249291201Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249307689Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249324387Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249340585Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:53:22.249356913Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"":"downstairs","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249378221Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249393399Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"0","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249410377Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"1","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249427315Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"client":"2","":"downstairs","session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249444333Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249459731Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"96c7dacc-cc15-464f-8958-53883d4444f7 is now active with session: 2a67742a-687e-4f21-9853-e15987bcd258","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249475699Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249492887Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371,"session_id":"2a67742a-687e-4f21-9853-e15987bcd258"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:22.249508566Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 233 len 8192 data: 3 2 02/30 Read block 208 len 28672 03/30 Flush 04/30 Write block 367 len 16384 data: 5 4 2 2 05/30 Read block 324 len 12288 06/30 Read block 307 len 12288 07/30 Read block 210 len 36864 08/30 Read block 411 len 36864 09/30 Read block 495 len 8192 10/30 Read block 126 len 4096 11/30 Read block 166 len 8192 12/30 Read block 1 len 24576 13/30 Read block 520 len 32768 14/30 Read block 510 len 24576 15/30 Read block 553 len 20480 16/30 Read block 489 len 12288 17/30 Write block 340 len 8192 data: 3 4 18/30 Write block 422 len 32768 data: 5 5 5 4 3 2 2 3 19/30 Read block 18 len 24576 20/30 Flush 21/30 Write block 551 len 4096 data: 7 22/30 Read block 189 len 36864 23/30 Read block 385 len 4096 24/30 Read block 379 len 12288 25/30 Write block 409 len 8192 data: 5 6 26/30 Read block 576 len 40960 27/30 Read block 371 len 4096 28/30 Write block 162 len 12288 data: 8 6 6 29/30 Write block 266 len 28672 data: 8 8 6 5 5 5 4 30/30 Write block 462 len 32768 data: 5 5 3 2 3 4 4 4 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:22.268963735Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1371} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 74 74 74 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 40 40 40 66 66 66 F F F 3 060-079 45 45 45 74 74 74 F F F 4 080-099 43 43 43 71 71 71 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 45 45 45 74 74 74 T T T 9 180-199 41 41 41 68 68 68 F F F 10 200-219 40 40 40 67 67 67 F F F 11 220-239 46 46 46 75 75 75 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 44 44 44 72 72 72 T T T 14 280-299 44 44 44 72 72 72 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 43 43 43 71 71 71 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 46 46 46 76 76 76 F F F 19 380-399 45 45 45 74 74 74 F F F 20 400-419 42 42 42 69 69 69 T T T 21 420-439 46 46 46 76 76 76 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 43 43 43 71 71 71 T T T 24 480-499 43 43 43 71 71 71 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 44 44 44 72 72 72 T T T 28 560-579 45 45 45 74 74 74 F F F 29 580-599 40 40 40 67 67 67 F F F Max gen: 46, Max flush: 76 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 74 74 74 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 40 40 40 66 66 66 F F F 3 060-079 45 45 45 74 74 74 F F F 4 080-099 43 43 43 71 71 71 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 47 47 47 77 77 77 F F F 9 180-199 41 41 41 68 68 68 F F F 10 200-219 40 40 40 67 67 67 F F F 11 220-239 46 46 46 75 75 75 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 47 47 47 77 77 77 F F F 14 280-299 44 44 44 72 72 72 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 43 43 43 71 71 71 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 46 46 46 76 76 76 F F F 19 380-399 45 45 45 74 74 74 F F F 20 400-419 47 47 47 77 77 77 F F F 21 420-439 46 46 46 76 76 76 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 47 47 47 77 77 77 F F F 24 480-499 43 43 43 71 71 71 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 47 47 47 77 77 77 F F F 28 560-579 45 45 45 74 74 74 F F F 29 580-599 40 40 40 67 67 67 F F F Max gen: 47, Max flush: 77 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.540185331Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.583797013Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.58441662Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.584454096Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.584468804Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.584489192Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379} {"msg":"Crucible af4200e7-d6d7-4aa6-ab4f-60009835d125 has session id: 2010e4f1-3a57-4998-a786-c2a946025461","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.584970476Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Upstairs opts: Upstairs UUID: af4200e7-d6d7-4aa6-ab4f-60009835d125, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.5850171Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Crucible stats registered with UUID: af4200e7-d6d7-4aa6-ab4f-60009835d125","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.585043617Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.615707179Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.615741125Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379} {"msg":"af4200e7-d6d7-4aa6-ab4f-60009835d125 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.615813027Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.615836254Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"0","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.615865331Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"0","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.615883158Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"1","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.615913045Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"1","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.615946921Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"2","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.615981197Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"2","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.616014543Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.616054738Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.616090954Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"ds_connection connected from Ok(127.0.0.1:39613)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.61612492Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"ds_connection connected from Ok(127.0.0.1:51267)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.616153947Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"ds_connection connected from Ok(127.0.0.1:57810)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.616233208Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.616958043Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"0","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.61698848Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"2","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.617050152Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"1","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.617658121Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"1","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.617803964Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"0","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.61783893Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"2","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.6181889Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618316785Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"[0]R flush_numbers[0..12]: [74, 58, 66, 74, 71, 70, 64, 64, 77, 68, 67, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618462917Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"[0]R generation[0..12]: [45, 35, 40, 45, 43, 42, 38, 38, 47, 41, 40, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618483395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618501733Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"[1]R flush_numbers[0..12]: [74, 58, 66, 74, 71, 70, 64, 64, 77, 68, 67, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618516361Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"[1]R generation[0..12]: [45, 35, 40, 45, 43, 42, 38, 38, 47, 41, 40, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618530429Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618544878Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"[2]R flush_numbers[0..12]: [74, 58, 66, 74, 71, 70, 64, 64, 77, 68, 67, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618558636Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"[2]R generation[0..12]: [45, 35, 40, 45, 43, 42, 38, 38, 47, 41, 40, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618572245Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618586183Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618599181Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.6186126Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:53:26.618625609Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618645506Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"":"downstairs","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618658765Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618670943Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"0","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618684222Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"1","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.61869911Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"client":"2","":"downstairs","session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618712578Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618724927Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"af4200e7-d6d7-4aa6-ab4f-60009835d125 is now active with session: 2010e4f1-3a57-4998-a786-c2a946025461","v":0,"name":"crucible","level":30,"time"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"2025-01-13T15:53:26.618738055Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618754963Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379,"session_id":"2010e4f1-3a57-4998-a786-c2a946025461"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:26.618767362Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 209 len 20480 data: 4 4 5 6 7 02/30 Write block 498 len 32768 data: 2 5 7 7 6 6 5 6 03/30 Read block 254 len 20480 04/30 Flush 05/30 Write block 463 len 12288 data: 6 4 3 06/30 Write block 178 len 8192 data: 5 5 07/30 Read block 130 len 32768 08/30 Read block 198 len 36864 09/30 Read block 388 len 24576 10/30 Read block 55 len 28672 11/30 Read block 489 len 20480 12/30 Read block 230 len 8192 13/30 Write block 555 len 32768 data: 5 5 5 5 4 3 3 2 14/30 Flush 15/30 Write block 373 len 36864 data: 4 4 4 3 4 3 3 4 5 16/30 Write block 474 len 32768 data: 4 4 4 4 6 6 5 4 17/30 Write block 43 len 4096 data: 7 18/30 Write block 330 len 20480 data: 3 3 4 2 2 19/30 Read block 390 len 8192 20/30 Read block 183 len 4096 21/30 Write block 52 len 40960 data: 6 5 5 5 4 3 3 3 3 3 22/30 Read block 383 len 40960 23/30 Read block 334 len 12288 24/30 Read block 348 len 12288 25/30 Write block 218 len 36864 data: 7 5 5 5 4 4 4 5 4 26/30 Read block 380 len 28672 27/30 Read block 8 len 24576 28/30 Read block 490 len 32768 29/30 Write block 492 len 12288 data: 4 2 2 30/30 Write block 89 len 20480 data: 9 8 10 9 8 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 9 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:26.638828601Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1379} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 74 74 74 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 40 40 40 66 66 66 T T T 3 060-079 45 45 45 74 74 74 T T T 4 080-099 43 43 43 71 71 71 T T T 5 100-119 42 42 42 70 70 70 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 48 48 48 79 79 79 F F F 9 180-199 41 41 41 68 68 68 F F F 10 200-219 48 48 48 78 78 78 T T T 11 220-239 46 46 46 75 75 75 T T T 12 240-259 37 37 37 61 61 61 F F F 13 260-279 47 47 47 77 77 77 F F F 14 280-299 44 44 44 72 72 72 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 43 43 43 71 71 71 T T T 17 340-359 46 46 46 76 76 76 F F F 18 360-379 46 46 46 76 76 76 T T T 19 380-399 45 45 45 74 74 74 T T T 20 400-419 47 47 47 77 77 77 F F F 21 420-439 46 46 46 76 76 76 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 48 48 48 79 79 79 T T T 24 480-499 48 48 48 78 78 78 T T T 25 500-519 48 48 48 78 78 78 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 48 48 48 79 79 79 F F F 28 560-579 48 48 48 79 79 79 F F F 29 580-599 40 40 40 67 67 67 F F F Max gen: 48, Max flush: 79 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 74 74 74 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 49 49 49 80 80 80 F F F 3 060-079 49 49 49 80 80 80 F F F 4 080-099 49 49 49 80 80 80 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 38 38 38 64 64 64 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 48 48 48 79 79 79 F F F 9 180-199 41 41 41 68 68 68 F F F 10 200-219 49 49 49 80 80 80 F F F 11 220-239 49 49 49 80 80 80 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 47 47 47 77 77 77 F F F 14 280-299 44 44 44 72 72 72 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 49 49 49 80 80 80 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 49 49 49 80 80 80 F F F 19 380-399 49 49 49 80 80 80 F F F 20 400-419 47 47 47 77 77 77 F F F 21 420-439 46 46 46 76 76 76 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 49 49 49 80 80 80 F F F 24 480-499 49 49 49 80 80 80 F F F 25 500-519 48 48 48 78 78 78 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 48 48 48 79 79 79 F F F 28 560-579 48 48 48 79 79 79 F F F 29 580-599 40 40 40 67 67 67 F F F Max gen: 49, Max flush: 80 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:30.940945192Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:30.986850305Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:30.987687227Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:30.987734432Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:30.98775337Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:30.987771028Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387} {"msg":"Crucible 9e4979ea-757d-44fb-b64a-0ed98feda408 has session id: 0178f670-432c-4596-8869-12576dac6ab5","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:30.988240073Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Upstairs opts: Upstairs UUID: 9e4979ea-757d-44fb-b64a-0ed98feda408, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:30.988294296Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Crucible stats registered with UUID: 9e4979ea-757d-44fb-b64a-0ed98feda408","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:30.988315624Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.01817131Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018209566Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387} {"msg":"9e4979ea-757d-44fb-b64a-0ed98feda408 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.01826011Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018280038Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"0","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018308924Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"0","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018332011Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"1","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018351299Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"1","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018368137Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"2","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018385805Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"2","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018457397Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018496842Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018525499Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"ds_connection connected from Ok(127.0.0.1:63336)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018553036Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46547)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018583152Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"ds_connection connected from Ok(127.0.0.1:60482)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.018613449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.019217828Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"2","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.019267113Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"0","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.019390938Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"1","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.01988417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"0","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.019919296Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"2","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.019954282Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"1","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020545503Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020582609Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"[0]R flush_numbers[0..12]: [74, 58, 80, 80, 80, 70, 64, 64, 79, 68, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020613705Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"[0]R generation[0..12]: [45, 35, 49, 49, 49, 42, 38, 38, 48, 41, 49, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020630034Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020644922Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"[1]R flush_numbers[0..12]: [74, 58, 80, 80, 80, 70, 64, 64, 79, 68, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.02065896Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"[1]R generation[0..12]: [45, 35, 49, 49, 49, 42, 38, 38, 48, 41, 49, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020672998Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020687067Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"[2]R flush_numbers[0..12]: [74, 58, 80, 80, 80, 70, 64, 64, 79, 68, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020700785Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"[2]R generation[0..12]: [45, 35, 49, 49, 49, 42, 38, 38, 48, 41, 49, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020714404Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020728402Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.02074139Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020754729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020772027Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020793694Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"":"downstairs","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"Wait for a query_work_queue command to finish before sending IO msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020827231Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020853557Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"0","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020881804Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"1","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020910401Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"client":"2","":"downstairs","session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020939257Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.020967524Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"9e4979ea-757d-44fb-b64a-0ed98feda408 is now active with session: 0178f670-432c-4596-8869-12576dac6ab5","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.02099804Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.021023717Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387,"session_id":"0178f670-432c-4596-8869-12576dac6ab5"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:31.021050184Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 39 len 4096 data: 3 02/30 Read block 4 len 24576 03/30 Write block 161 len 28672 data: 8 9 7 7 5 7 7 04/30 Read block 586 len 20480 05/30 Write block 167 len 20480 data: 8 8 7 7 5 06/30 Read block 251 len 16384 07/30 Read block 451 len 8192 08/30 Read block 517 len 12288 09/30 Flush 10/30 Write block 177 len 20480 data: 6 6 6 6 5 11/30 Write block 130 len 20480 data: 2 2 2 3 5 12/30 Read block 212 len 28672 13/30 Write block 564 len 24576 data: 2 2 2 3 3 5 14/30 Read block 487 len 20480 15/30 Write block 19 len 4096 data: 6 16/30 Read block 90 len 36864 17/30 Read block 36 len 36864 18/30 Read block 340 len 8192 19/30 Read block 515 len 40960 20/30 Read block 541 len 24576 21/30 Read block 94 len 8192 22/30 Read block 469 len 36864 23/30 Write block 158 len 16384 data: 7 7 8 9 24/30 Read block 521 len 36864 25/30 Read block 235 len 24576 26/30 Read block 492 len 12288 27/30 Read block 515 len 12288 28/30 Write block 551 len 4096 data: 8 29/30 Read block 400 len 8192 30/30 Write block 70 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 5 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:31.040057547Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1387} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 74 74 74 T T T 1 020-039 50 50 50 81 81 81 F F F 2 040-059 49 49 49 80 80 80 F F F 3 060-079 49 49 49 80 80 80 T T T 4 080-099 49 49 49 80 80 80 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 38 38 38 64 64 64 T T T 7 140-159 38 38 38 64 64 64 T T T 8 160-179 50 50 50 81 81 81 T T T 9 180-199 41 41 41 68 68 68 T T T 10 200-219 49 49 49 80 80 80 F F F 11 220-239 49 49 49 80 80 80 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 47 47 47 77 77 77 F F F 14 280-299 44 44 44 72 72 72 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 49 49 49 80 80 80 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 49 49 49 80 80 80 F F F 19 380-399 49 49 49 80 80 80 F F F 20 400-419 47 47 47 77 77 77 F F F 21 420-439 46 46 46 76 76 76 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 49 49 49 80 80 80 F F F 24 480-499 49 49 49 80 80 80 F F F 25 500-519 48 48 48 78 78 78 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 48 48 48 79 79 79 T T T 28 560-579 48 48 48 79 79 79 T T T 29 580-599 40 40 40 67 67 67 F F F Max gen: 50, Max flush: 81 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 51 51 51 82 82 82 F F F 1 020-039 50 50 50 81 81 81 F F F 2 040-059 49 49 49 80 80 80 F F F 3 060-079 51 51 51 82 82 82 F F F 4 080-099 49 49 49 80 80 80 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 51 51 51 82 82 82 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 51 51 51 82 82 82 F F F 9 180-199 51 51 51 82 82 82 F F F 10 200-219 49 49 49 80 80 80 F F F 11 220-239 49 49 49 80 80 80 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 47 47 47 77 77 77 F F F 14 280-299 44 44 44 72 72 72 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 49 49 49 80 80 80 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 49 49 49 80 80 80 F F F 19 380-399 49 49 49 80 80 80 F F F 20 400-419 47 47 47 77 77 77 F F F 21 420-439 46 46 46 76 76 76 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 49 49 49 80 80 80 F F F 24 480-499 49 49 49 80 80 80 F F F 25 500-519 48 48 48 78 78 78 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 51 51 51 82 82 82 F F F 29 580-599 40 40 40 67 67 67 F F F Max gen: 51, Max flush: 82 Begin loop 26 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.335053779Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.379411304Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.380194562Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.380247536Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.380273833Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.3803007Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395} {"msg":"Crucible 06c3e1be-a21d-4283-9699-5b6b63603988 has session id: 961da78d-533d-4e80-8b3a-4c8cd6fcc0c5","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.380746008Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Upstairs opts: Upstairs UUID: 06c3e1be-a21d-4283-9699-5b6b63603988, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.380781173Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Crucible stats registered with UUID: 06c3e1be-a21d-4283-9699-5b6b63603988","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.380799871Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.410903259Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.410934885Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395} {"msg":"06c3e1be-a21d-4283-9699-5b6b63603988 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.410966591Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.41098288Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"0","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.411010776Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"0","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.411028424Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"1","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.411046152Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"1","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.41106257Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"2","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.411080008Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"2","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.411096276Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.411113334Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.411130462Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:39630)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.411353856Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:33268)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.41140272Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:32875)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.411442356Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.412019668Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"1","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.412053695Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"0","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.412296736Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"2","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.412764652Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"0","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.412850791Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"1","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.412965238Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"2","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413315467Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413428674Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"[0]R flush_numbers[0..12]: [82, 81, 80, 82, 80, 70, 82, 82, 82, 82, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413555149Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"[0]R generation[0..12]: [51, 50, 49, 51, 49, 42, 51, 51, 51, 51, 49, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413575637Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413590675Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"[1]R flush_numbers[0..12]: [82, 81, 80, 82, 80, 70, 82, 82, 82, 82, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413604783Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"[1]R generation[0..12]: [51, 50, 49, 51, 49, 42, 51, 51, 51, 51, 49, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413618772Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.41363644Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"[2]R flush_numbers[0..12]: [82, 81, 80, 82, 80, 70, 82, 82, 82, 82, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413650678Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"[2]R generation[0..12]: [51, 50, 49, 51, 49, 42, 51, 51, 51, 51, 49, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413664367Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413678395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413691353Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413704672Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:53:35.41371761Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413736718Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"":"downstairs","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413749966Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413761995Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"0","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413775274Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"1","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413788482Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"client":"2","":"downstairs","session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413801851Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413814109Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"06c3e1be-a21d-4283-9699-5b6b63603988 is now active with session: 961da78d-533d-4e80-8b3a-4c8cd6fcc0c5","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413826948Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "time":"2025-01-13T15:53:35.413838946Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395,"session_id":"961da78d-533d-4e80-8b3a-4c8cd6fcc0c5"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:35.413858204Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 378 len 36864 data: 4 4 5 6 5 5 5 5 5 03/30 Read block 16 len 8192 04/30 Flush 05/30 Write block 585 len 40960 data: 6 4 5 4 5 4 6 6 6 6 06/30 Read block 19 len 36864 07/30 Write block 368 len 24576 data: 5 3 3 4 4 5 08/30 Read block 265 len 24576 09/30 Read block 22 len 16384 10/30 Write block 260 len 40960 data: 6 6 6 6 7 8 9 9 7 6 11/30 Write block 488 len 36864 data: 6 6 5 5 5 3 3 2 2 12/30 Write block 62 len 32768 data: 3 4 5 4 5 6 6 5 13/30 Flush 14/30 Write block 274 len 40960 data: 5 5 6 6 5 6 6 5 5 4 15/30 Read block 394 len 36864 16/30 Read block 125 len 28672 17/30 Write block 207 len 36864 data: 4 4 5 5 6 7 8 7 6 18/30 Read block 279 len 8192 19/30 Read block 336 len 40960 20/30 Flush 21/30 Write block 22 len 32768 data: 4 3 3 3 4 4 4 4 22/30 Read block 95 len 4096 23/30 Read block 58 len 12288 24/30 Flush 25/30 Write block 42 len 12288 data: 5 8 7 26/30 Read block 210 len 20480 27/30 Write block 404 len 20480 data: 5 6 6 5 6 28/30 Read block 483 len 24576 29/30 Read block 183 len 4096 30/30 Read block 202 len 16384 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 6 5 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:35.436292446Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1395} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 51 51 51 82 82 82 F F F 1 020-039 52 52 52 87 87 87 F F F 2 040-059 49 49 49 80 80 80 T T T 3 060-079 52 52 52 85 85 85 F F F 4 080-099 49 49 49 80 80 80 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 51 51 51 82 82 82 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 51 51 51 82 82 82 F F F 9 180-199 51 51 51 82 82 82 F F F 10 200-219 52 52 52 86 86 86 F F F 11 220-239 49 49 49 80 80 80 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 52 52 52 86 86 86 F F F 14 280-299 52 52 52 86 86 86 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 49 49 49 80 80 80 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 52 52 52 85 85 85 F F F 19 380-399 52 52 52 84 84 84 F F F 20 400-419 47 47 47 77 77 77 T T T 21 420-439 46 46 46 76 76 76 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 49 49 49 80 80 80 F F F 24 480-499 52 52 52 85 85 85 F F F 25 500-519 48 48 48 78 78 78 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 51 51 51 82 82 82 F F F 29 580-599 52 52 52 85 85 85 F F F Max gen: 52, Max flush: 87 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 51 51 51 82 82 82 F F F 1 020-039 52 52 52 87 87 87 F F F 2 040-059 53 53 53 88 88 88 F F F 3 060-079 52 52 52 85 85 85 F F F 4 080-099 49 49 49 80 80 80 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 51 51 51 82 82 82 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 51 51 51 82 82 82 F F F 9 180-199 51 51 51 82 82 82 F F F 10 200-219 52 52 52 86 86 86 F F F 11 220-239 49 49 49 80 80 80 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 52 52 52 86 86 86 F F F 14 280-299 52 52 52 86 86 86 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 49 49 49 80 80 80 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 52 52 52 85 85 85 F F F 19 380-399 52 52 52 84 84 84 F F F 20 400-419 53 53 53 88 88 88 F F F 21 420-439 46 46 46 76 76 76 F F F 22 440-459 41 41 41 68 68 68 F F F 23 460-479 49 49 49 80 80 80 F F F 24 480-499 52 52 52 85 85 85 F F F 25 500-519 48 48 48 78 78 78 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 51 51 51 82 82 82 F F F 29 580-599 52 52 52 85 85 85 F F F Max gen: 53, Max flush: 88 Begin loop 27 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.697959417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.742449776Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.743339022Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.743378268Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.743393496Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.743407644Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403} {"msg":"Crucible f447a11a-cc43-4d6b-aed7-642935831ee3 has session id: d58da458-f99f-43a1-bcdf-d91acd79f1e1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.743864661Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"Upstairs opts: Upstairs UUID: f447a11a-cc43-4d6b-aed7-642935831ee3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.743912685Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"Crucible stats registered with UUID: f447a11a-cc43-4d6b-aed7-642935831ee3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.743938312Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.773940731Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.773996065Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403} {"msg":"f447a11a-cc43-4d6b-aed7-642935831ee3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.774013213Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.774030261Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"0","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.774060237Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"0","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.774078485Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"1","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.774096363Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"1","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.774112971Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"2","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.774130389Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"2","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.774146987Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.774164505Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.774186103Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:39196)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.7743002Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51806)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.774360602Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46791)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.774400748Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.77506771Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"1","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.775098796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"0","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.775244059Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"2","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.77575596Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"0","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.775796755Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"1","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.775951187Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"2","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776287457Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776397024Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"[0]R flush_numbers[0..12]: [82, 87, 88, 85, 80, 70, 82, 82, 82, 82, 86, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776529219Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"[0]R generation[0..12]: [51, 52, 53, 52, 49, 42, 51, 51, 51, 51, 52, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776551276Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776566944Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"[1]R flush_numbers[0..12]: [82, 87, 88, 85, 80, 70, 82, 82, 82, 82, 86, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776584982Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"[1]R generation[0..12]: [51, 52, 53, 52, 49, 42, 51, 51, 51, 51, 52, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.77659912Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776613309Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"[2]R flush_numbers[0..12]: [82, 87, 88, 85, 80, 70, 82, 82, 82, 82, 86, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776627657Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"[2]R generation[0..12]: [51, 52, 53, 52, 49, 42, 51, 51, 51, 51, 52, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776651944Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776681251Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776712827Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776740694Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776766631Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776793548Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"":"downstairs","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776810066Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776830324Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"0","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.77685731Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"1","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776885077Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"client":"2","":"downstairs","session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776904855Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776922233Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"f447a11a-cc43-4d6b-aed7-642935831ee3 is now active with session: d58da458-f99f-43a1-bcdf-d91acd79f1e1","v":0,"name":"crucible",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "level":30,"time":"2025-01-13T15:53:39.776949239Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776971017Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403,"session_id":"d58da458-f99f-43a1-bcdf-d91acd79f1e1"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:39.776990805Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 583 len 40960 data: 7 6 7 5 6 5 6 5 7 7 02/30 Read block 162 len 32768 03/30 Read block 510 len 24576 04/30 Read block 459 len 4096 05/30 Read block 141 len 28672 06/30 Flush 07/30 Write block 193 len 40960 data: 4 5 4 3 3 4 4 4 4 4 08/30 Read block 559 len 24576 09/30 Read block 277 len 40960 10/30 Flush 11/30 Write block 397 len 12288 data: 6 5 5 12/30 Write block 508 len 8192 data: 6 5 13/30 Write block 436 len 36864 data: 5 5 5 3 3 3 3 4 4 14/30 Read block 399 len 28672 15/30 Read block 326 len 28672 16/30 Read block 146 len 40960 17/30 Read block 384 len 28672 18/30 Read block 417 len 12288 19/30 Read block 52 len 8192 20/30 Write block 77 len 16384 data: 2 3 2 2 21/30 Read block 305 len 32768 22/30 Read block 414 len 28672 23/30 Write block 18 len 36864 data: 6 7 6 4 5 4 4 4 5 24/30 Write block 175 len 40960 data: 3 4 7 7 7 7 6 4 5 6 25/30 Read block 141 len 8192 26/30 Read block 460 len 24576 27/30 Read block 495 len 40960 28/30 Write block 89 len 32768 data: 10 9 11 10 9 7 7 5 29/30 Read block 506 len 20480 30/30 Read block 300 len 12288 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 3 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:39.79896703Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1403} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 51 51 51 82 82 82 T T T 1 020-039 52 52 52 87 87 87 T T T 2 040-059 53 53 53 88 88 88 F F F 3 060-079 52 52 52 85 85 85 T T T 4 080-099 49 49 49 80 80 80 T T T 5 100-119 42 42 42 70 70 70 F F F 6 120-139 51 51 51 82 82 82 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 51 51 51 82 82 82 T T T 9 180-199 54 54 54 90 90 90 T T T 10 200-219 54 54 54 90 90 90 F F F 11 220-239 49 49 49 80 80 80 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 52 52 52 86 86 86 F F F 14 280-299 52 52 52 86 86 86 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 49 49 49 80 80 80 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 52 52 52 85 85 85 F F F 19 380-399 52 52 52 84 84 84 T T T 20 400-419 53 53 53 88 88 88 F F F 21 420-439 46 46 46 76 76 76 T T T 22 440-459 41 41 41 68 68 68 T T T 23 460-479 49 49 49 80 80 80 F F F 24 480-499 52 52 52 85 85 85 F F F 25 500-519 48 48 48 78 78 78 T T T 26 520-539 44 44 44 72 72 72 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 51 51 51 82 82 82 F F F 29 580-599 54 54 54 89 89 89 F F F Max gen: 54, Max flush: 90 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 91 91 91 F F F 1 020-039 55 55 55 91 91 91 F F F 2 040-059 53 53 53 88 88 88 F F F 3 060-079 55 55 55 91 91 91 F F F 4 080-099 55 55 55 91 91 91 F F F 5 100-119 42 42 42 70 70 70 F F F 6 120-139 51 51 51 82 82 82 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 55 55 55 91 91 91 F F F 9 180-199 55 55 55 91 91 91 F F F 10 200-219 54 54 54 90 90 90 F F F 11 220-239 49 49 49 80 80 80 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 52 52 52 86 86 86 F F F 14 280-299 52 52 52 86 86 86 F F F 15 300-319 40 40 40 67 67 67 F F F 16 320-339 49 49 49 80 80 80 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 52 52 52 85 85 85 F F F 19 380-399 55 55 55 91 91 91 F F F 20 400-419 53 53 53 88 88 88 F F F 21 420-439 55 55 55 91 91 91 F F F 22 440-459 55 55 55 91 91 91 F F F 23 460-479 49 49 49 80 80 80 F F F 24 480-499 52 52 52 85 85 85 F F F 25 500-519 55 55 55 91 91 91 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 51 51 51 82 82 82 F F F 28 560-579 51 51 51 82 82 82 F F F 29 580-599 54 54 54 89 89 89 F F F Max gen: 55, Max flush: 91 Begin loop 28 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.109957438Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:44.153163436Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.153554721Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.184692377Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.185331683Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.185370658Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.185385056Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.185399825Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411} {"msg":"Crucible db158ce6-1d6e-49f1-8498-53becc5d4eaf has session id: d3993845-6e74-432b-9456-94f376c84601","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.18587359Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Upstairs opts: Upstairs UUID: db158ce6-1d6e-49f1-8498-53becc5d4eaf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.185911535Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Crucible stats registered with UUID: db158ce6-1d6e-49f1-8498-53becc5d4eaf","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.185928893Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216072916Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216110291Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411} {"msg":"db158ce6-1d6e-49f1-8498-53becc5d4eaf active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216132319Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216148767Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"0","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216183453Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"0","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216202071Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"1","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216220089Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"1","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216236787Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"2","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216254274Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"2","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216270603Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216287721Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216310488Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"ds_connection connected from Ok(127.0.0.1:58927)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216354483Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"ds_connection connected from Ok(127.0.0.1:63762)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216399388Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"ds_connection connected from Ok(127.0.0.1:40138)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.216438033Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.217186336Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"2","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.217227231Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"1","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.217415719Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"0","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.217893803Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"1","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.217942218Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"2","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.217978353Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"0","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.21843226Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218565375Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"[0]R flush_numbers[0..12]: [91, 91, 88, 91, 91, 70, 82, 82, 91, 91, 90, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.21860704Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"[0]R generation[0..12]: [55, 55, 53, 55, 55, 42, 51, 51, 55, 55, 54, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218624148Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218639086Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"[1]R flush_numbers[0..12]: [91, 91, 88, 91, 91, 70, 82, 82, 91, 91, 90, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218653974Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"[1]R generation[0..12]: [55, 55, 53, 55, 55, 42, 51, 51, 55, 55, 54, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218667923Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218682221Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"[2]R flush_numbers[0..12]: [91, 91, 88, 91, 91, 70, 82, 82, 91, 91, 90, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.21869607Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"[2]R generation[0..12]: [55, 55, 53, 55, 55, 42, 51, 51, 55, 55, 54, 49]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218709668Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218723477Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218740435Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218754553Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Next flush: 92","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO :30,"time":"2025-01-13T15:53:44.21877578Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218789519Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"":"downstairs","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218803017Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218815156Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"0","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218828624Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"1","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218841653Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"client":"2","":"downstairs","session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218854811Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.21886705Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"db158ce6-1d6e-49f1-8498-53becc5d4eaf is now active with session: d3993845-6e74-432b-9456-94f376c84601","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218879878Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218891897Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411,"session_id":"d3993845-6e74-432b-9456-94f376c84601"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:44.218904215Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 297 len 28672 data: 4 4 4 4 4 5 5 02/30 Read block 157 len 16384 03/30 Write block 43 len 32768 data: 9 8 7 8 6 6 6 7 04/30 Write block 337 len 12288 data: 4 4 4 05/30 Read block 77 len 32768 06/30 Flush 07/30 Write block 587 len 24576 data: 7 6 7 6 8 8 08/30 Flush 09/30 Write block 162 len 36864 data: 10 8 8 6 8 9 9 8 8 10/30 Write block 89 len 40960 data: 11 10 12 11 10 8 8 6 5 5 11/30 Write block 116 len 32768 data: 7 4 3 4 4 4 5 6 12/30 Write block 550 len 24576 data: 6 9 7 7 4 6 13/30 Read block 110 len 36864 14/30 Read block 117 len 12288 15/30 Read block 572 len 12288 16/30 Write block 541 len 4096 data: 5 17/30 Read block 118 len 24576 18/30 Read block 378 len 40960 19/30 Read block 357 len 20480 20/30 Read block 506 len 16384 21/30 Read block 26 len 8192 22/30 Read block 356 len 40960 23/30 Write block 215 len 28672 data: 7 7 7 8 6 6 6 24/30 Write block 96 len 16384 data: 7 6 6 5 25/30 Write block 389 len 12288 data: 7 7 7 26/30 Write block 1 len 4096 data: 3 27/30 Read block 316 len 20480 28/30 Read block 370 len 4096 29/30 Write block 516 len 16384 data: 7 6 5 5 30/30 Read block 407 len 24576 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 9 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 21 22 21 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:44.238698216Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1411} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 91 91 91 T T T 1 020-039 55 55 55 91 91 91 F F F 2 040-059 56 56 56 92 92 92 F F F 3 060-079 55 55 55 91 91 91 F F F 4 080-099 55 55 55 91 91 91 T T T 5 100-119 42 42 42 70 70 70 T T T 6 120-139 51 51 51 82 82 82 T T T 7 140-159 51 51 51 82 82 82 F F F 8 160-179 55 55 55 91 91 91 T T T 9 180-199 55 55 55 91 91 91 F F F 10 200-219 54 54 54 90 90 90 T T T 11 220-239 49 49 49 80 80 80 T T T 12 240-259 37 37 37 61 61 61 F F F 13 260-279 52 52 52 86 86 86 F F F 14 280-299 56 56 56 92 92 92 F F F 15 300-319 56 56 56 92 92 92 F F F 16 320-339 56 56 56 92 92 92 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 52 52 52 85 85 85 F F F 19 380-399 55 55 55 91 91 91 T T T 20 400-419 53 53 53 88 88 88 F F F 21 420-439 55 55 55 91 91 91 F F F 22 440-459 55 55 55 91 91 91 F F F 23 460-479 49 49 49 80 80 80 F F F 24 480-499 52 52 52 85 85 85 F F F 25 500-519 55 55 55 91 91 91 T T T 26 520-539 44 44 44 72 72 72 F F F 27 540-559 51 51 51 82 82 82 T T T 28 560-579 51 51 51 82 82 82 F F F 29 580-599 56 56 56 93 93 93 F F F Max gen: 56, Max flush: 93 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 94 94 94 F F F 1 020-039 55 55 55 91 91 91 F F F 2 040-059 56 56 56 92 92 92 F F F 3 060-079 55 55 55 91 91 91 F F F 4 080-099 57 57 57 94 94 94 F F F 5 100-119 57 57 57 94 94 94 F F F 6 120-139 57 57 57 94 94 94 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 57 57 57 94 94 94 F F F 9 180-199 55 55 55 91 91 91 F F F 10 200-219 57 57 57 94 94 94 F F F 11 220-239 57 57 57 94 94 94 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 52 52 52 86 86 86 F F F 14 280-299 56 56 56 92 92 92 F F F 15 300-319 56 56 56 92 92 92 F F F 16 320-339 56 56 56 92 92 92 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 52 52 52 85 85 85 F F F 19 380-399 57 57 57 94 94 94 F F F 20 400-419 53 53 53 88 88 88 F F F 21 420-439 55 55 55 91 91 91 F F F 22 440-459 55 55 55 91 91 91 F F F 23 460-479 49 49 49 80 80 80 F F F 24 480-499 52 52 52 85 85 85 F F F 25 500-519 57 57 57 94 94 94 F F F 26 520-539 44 44 44 72 72 72 F F F 27 540-559 57 57 57 94 94 94 F F F 28 560-579 51 51 51 82 82 82 F F F 29 580-599 56 56 56 93 93 93 F F F Max gen: 57, Max flush: 94 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.585619029Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:48.630270829Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.63086142Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.662951525Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.663891356Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.66394011Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.663958008Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.663981415Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419} {"msg":"Crucible 2c272b03-0180-4f27-a366-1ace8289a167 has session id: d8b0a8a2-1fba-43ef-a52e-d91fef8c6884","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.66445449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Upstairs opts: Upstairs UUID: 2c272b03-0180-4f27-a366-1ace8289a167, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.664501955Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Crucible stats registered with UUID: 2c272b03-0180-4f27-a366-1ace8289a167","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.664523642Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695381531Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695447794Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419} {"msg":"2c272b03-0180-4f27-a366-1ace8289a167 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695469681Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695495778Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"0","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695544312Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"0","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695577729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"1","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695612944Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"1","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695645211Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"2","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695679566Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"2","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695707643Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.69573339Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695757737Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"ds_connection connected from Ok(127.0.0.1:42906)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695777845Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"ds_connection connected from Ok(127.0.0.1:56203)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.695839908Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"ds_connection connected from Ok(127.0.0.1:57577)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.696053373Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.696719115Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"1","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.69676261Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"2","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.696908823Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"0","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.697355061Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"1","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.697396336Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"2","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.697437652Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"0","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.697983938Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698014744Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"[0]R flush_numbers[0..12]: [94, 91, 92, 91, 94, 94, 94, 82, 94, 91, 94, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698104794Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"[0]R generation[0..12]: [57, 55, 56, 55, 57, 57, 57, 51, 57, 55, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698125841Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698146209Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"[1]R flush_numbers[0..12]: [94, 91, 92, 91, 94, 94, 94, 82, 94, 91, 94, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698161527Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"[1]R generation[0..12]: [57, 55, 56, 55, 57, 57, 57, 51, 57, 55, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698176215Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698190354Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"[2]R flush_numbers[0..12]: [94, 91, 92, 91, 94, 94, 94, 82, 94, 91, 94, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698204312Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"[2]R generation[0..12]: [57, 55, 56, 55, 57, 57, 57, 51, 57, 55, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698235358Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698249857Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698263145Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698276454Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698289662Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4"Wait for a query_work_queue command to finish before sending IO ,"pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698310539Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"":"downstairs","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698324128Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698336336Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"0","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698349905Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"1","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698363144Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"client":"2","":"downstairs","session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698379782Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.6983932Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"2c272b03-0180-4f27-a366-1ace8289a167 is now active with session: d8b0a8a2-1fba-43ef-a52e-d91fef8c6884","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698406339Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698418497Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419,"session_id":"d8b0a8a2-1fba-43ef-a52e-d91fef8c6884"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:48.698430785Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 168 len 4096 data: 10 04/30 Read block 234 len 40960 05/30 Read block 6 len 32768 06/30 Read block 101 len 8192 07/30 Flush 08/30 Write block 195 len 28672 data: 5 4 4 5 5 5 5 09/30 Write block 137 len 20480 data: 6 5 5 7 7 10/30 Read block 517 len 36864 11/30 Write block 207 len 40960 data: 5 5 6 6 7 8 9 8 8 8 12/30 Write block 322 len 4096 data: 6 13/30 Write block 497 len 32768 data: 2 3 6 8 8 7 7 6 14/30 Write block 332 len 12288 data: 5 3 3 15/30 Read block 46 len 32768 16/30 Read block 118 len 12288 17/30 Write block 51 len 36864 data: 6 7 6 6 6 5 4 4 4 18/30 Flush 19/30 Flush 20/30 Write block 143 len 40960 data: 5 5 4 2 3 3 3 3 4 4 21/30 Flush 22/30 Write block 306 len 8192 data: 3 3 23/30 Write block 107 len 4096 data: 4 24/30 Read block 47 len 40960 25/30 Write block 562 len 36864 data: 3 2 3 3 3 4 4 6 5 26/30 Write block 252 len 40960 data: 5 5 5 5 4 4 4 5 7 7 27/30 Write block 325 len 36864 data: 6 4 3 4 4 4 4 6 4 28/30 Write block 537 len 16384 data: 8 6 6 6 29/30 Read block 563 len 16384 30/30 Write block 82 len 8192 data: 5 5 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:48.717184787Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1419} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 94 94 94 F F F 1 020-039 55 55 55 91 91 91 F F F 2 040-059 58 58 58 98 98 98 F F F 3 060-079 55 55 55 91 91 91 F F F 4 080-099 57 57 57 94 94 94 T T T 5 100-119 57 57 57 94 94 94 T T T 6 120-139 58 58 58 98 98 98 F F F 7 140-159 58 58 58 100 100 100 F F F 8 160-179 58 58 58 97 97 97 F F F 9 180-199 58 58 58 98 98 98 F F F 10 200-219 58 58 58 98 98 98 F F F 11 220-239 57 57 57 94 94 94 F F F 12 240-259 37 37 37 61 61 61 T T T 13 260-279 52 52 52 86 86 86 T T T 14 280-299 56 56 56 92 92 92 F F F 15 300-319 56 56 56 92 92 92 T T T 16 320-339 58 58 58 98 98 98 T T T 17 340-359 46 46 46 76 76 76 F F F 18 360-379 52 52 52 85 85 85 F F F 19 380-399 57 57 57 94 94 94 F F F 20 400-419 53 53 53 88 88 88 F F F 21 420-439 55 55 55 91 91 91 F F F 22 440-459 55 55 55 91 91 91 F F F 23 460-479 49 49 49 80 80 80 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 58 58 58 98 98 98 F F F 26 520-539 44 44 44 72 72 72 T T T 27 540-559 57 57 57 94 94 94 T T T 28 560-579 51 51 51 82 82 82 T T T 29 580-599 56 56 56 93 93 93 F F F Max gen: 58, Max flush: 100 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 94 94 94 F F F 1 020-039 55 55 55 91 91 91 F F F 2 040-059 58 58 58 98 98 98 F F F 3 060-079 55 55 55 91 91 91 F F F 4 080-099 59 59 59 101 101 101 F F F 5 100-119 59 59 59 101 101 101 F F F 6 120-139 58 58 58 98 98 98 F F F 7 140-159 58 58 58 100 100 100 F F F 8 160-179 58 58 58 97 97 97 F F F 9 180-199 58 58 58 98 98 98 F F F 10 200-219 58 58 58 98 98 98 F F F 11 220-239 57 57 57 94 94 94 F F F 12 240-259 59 59 59 101 101 101 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 56 56 56 92 92 92 F F F 15 300-319 59 59 59 101 101 101 F F F 16 320-339 59 59 59 101 101 101 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 52 52 52 85 85 85 F F F 19 380-399 57 57 57 94 94 94 F F F 20 400-419 53 53 53 88 88 88 F F F 21 420-439 55 55 55 91 91 91 F F F 22 440-459 55 55 55 91 91 91 F F F 23 460-479 49 49 49 80 80 80 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 58 58 58 98 98 98 F F F 26 520-539 59 59 59 101 101 101 F F F 27 540-559 59 59 59 101 101 101 F F F 28 560-579 59 59 59 101 101 101 F F F 29 580-599 56 56 56 93 93 93 F F F Max gen: 59, Max flush: 101 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.051953148Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.098534623Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.099411231Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.099453106Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.099468224Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.099482993Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427} {"msg":"Crucible 58eb2c5e-48ff-4e0f-bae4-0ccdc0b56cdc has session id: bef6c24e-78f0-4532-a038-59333928cf6c","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.099875977Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Upstairs opts: Upstairs UUID: 58eb2c5e-48ff-4e0f-bae4-0ccdc0b56cdc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.099940849Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Crucible stats registered with UUID: 58eb2c5e-48ff-4e0f-bae4-0ccdc0b56cdc","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.099967046Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.130688201Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.130748304Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427} {"msg":"58eb2c5e-48ff-4e0f-bae4-0ccdc0b56cdc active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.130764082Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.13078037Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"0","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.130809297Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"0","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.130826685Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"1","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.130844503Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"1","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.130924644Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"2","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.130978477Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"2","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.131026712Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.131074496Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.131093794Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35144)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.131111912Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:53335)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.131200352Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33962)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.131290631Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.132096087Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"2","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.132126853Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"0","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.132287285Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"1","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.132833251Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"2","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.132865577Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"0","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.132996522Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"1","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.13344236Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133468807Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"[0]R flush_numbers[0..12]: [94, 91, 98, 91, 101, 101, 98, 100, 97, 98, 98, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133586263Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"[0]R generation[0..12]: [57, 55, 58, 55, 59, 59, 58, 58, 58, 58, 58, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.13360768Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133622459Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"[1]R flush_numbers[0..12]: [94, 91, 98, 91, 101, 101, 98, 100, 97, 98, 98, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133636487Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"[1]R generation[0..12]: [57, 55, 58, 55, 59, 59, 58, 58, 58, 58, 58, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133650386Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133664524Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"[2]R flush_numbers[0..12]: [94, 91, 98, 91, 101, 101, 98, 100, 97, 98, 98, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133678312Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"[2]R generation[0..12]: [57, 55, 58, 55, 59, 59, 58, 58, 58, 58, 58, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133691861Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133705759Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133718818Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133732176Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133745115Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133758203Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"":"downstairs","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133775481Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133788339Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"0","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133808507Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"1","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133821966Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"client":"2","":"downstairs","session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133835354Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133847643Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"58eb2c5e-48ff-4e0f-bae4-0ccdc0b56cdc is now active with session: bef6c24e-78f0-4532-a038-59333928cf6c","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133875919Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133888448Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427,"session_id":"bef6c24e-78f0-4532-a038-59333928cf6c"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:53.133900806Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 143 len 28672 data: 6 6 5 3 4 4 4 02/30 Write block 49 len 28672 data: 7 8 7 8 7 7 7 03/30 Write block 204 len 4096 data: 4 04/30 Write block 587 len 20480 data: 8 7 8 7 9 05/30 Flush 06/30 Write block 144 len 32768 data: 7 6 4 5 5 5 4 5 07/30 Write block 382 len 12288 data: 6 6 6 08/30 Write block 62 len 8192 data: 4 5 09/30 Read block 12 len 36864 10/30 Flush 11/30 Write block 191 len 28672 data: 5 4 5 6 6 5 5 12/30 Read block 519 len 16384 13/30 Flush 14/30 Write block 235 len 16384 data: 3 3 3 4 15/30 Read block 498 len 20480 16/30 Read block 118 len 24576 17/30 Write block 466 len 40960 data: 4 5 5 5 3 3 3 3 5 5 18/30 Flush 19/30 Write block 228 len 32768 data: 5 5 6 5 6 4 3 4 20/30 Write block 250 len 24576 data: 5 5 6 6 6 6 21/30 Write block 16 len 12288 data: 5 5 7 22/30 Write block 189 len 12288 data: 6 5 6 23/30 Write block 320 len 8192 data: 5 5 24/30 Read block 361 len 24576 25/30 Read block 386 len 4096 26/30 Read block 563 len 28672 27/30 Write block 408 len 28672 data: 7 6 7 5 6 7 6 28/30 Read block 159 len 36864 29/30 Write block 396 len 4096 data: 3 30/30 Read block 490 len 8192 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 8 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:53.152306509Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1427} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 94 94 94 T T T 1 020-039 55 55 55 91 91 91 F F F 2 040-059 60 60 60 102 102 102 F F F 3 060-079 60 60 60 103 103 103 F F F 4 080-099 59 59 59 101 101 101 F F F 5 100-119 59 59 59 101 101 101 F F F 6 120-139 58 58 58 98 98 98 F F F 7 140-159 60 60 60 103 103 103 F F F 8 160-179 58 58 58 97 97 97 F F F 9 180-199 60 60 60 104 104 104 T T T 10 200-219 60 60 60 102 102 102 F F F 11 220-239 60 60 60 105 105 105 T T T 12 240-259 59 59 59 101 101 101 T T T 13 260-279 59 59 59 101 101 101 F F F 14 280-299 56 56 56 92 92 92 F F F 15 300-319 59 59 59 101 101 101 F F F 16 320-339 59 59 59 101 101 101 T T T 17 340-359 46 46 46 76 76 76 F F F 18 360-379 52 52 52 85 85 85 F F F 19 380-399 60 60 60 103 103 103 T T T 20 400-419 53 53 53 88 88 88 T T T 21 420-439 55 55 55 91 91 91 F F F 22 440-459 55 55 55 91 91 91 F F F 23 460-479 60 60 60 105 105 105 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 58 58 58 98 98 98 F F F 26 520-539 59 59 59 101 101 101 F F F 27 540-559 59 59 59 101 101 101 F F F 28 560-579 59 59 59 101 101 101 F F F 29 580-599 60 60 60 102 102 102 F F F Max gen: 60, Max flush: 105 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 106 106 106 F F F 1 020-039 55 55 55 91 91 91 F F F 2 040-059 60 60 60 102 102 102 F F F 3 060-079 60 60 60 103 103 103 F F F 4 080-099 59 59 59 101 101 101 F F F 5 100-119 59 59 59 101 101 101 F F F 6 120-139 58 58 58 98 98 98 F F F 7 140-159 60 60 60 103 103 103 F F F 8 160-179 58 58 58 97 97 97 F F F 9 180-199 61 61 61 106 106 106 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 61 61 61 106 106 106 F F F 12 240-259 61 61 61 106 106 106 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 56 56 56 92 92 92 F F F 15 300-319 59 59 59 101 101 101 F F F 16 320-339 61 61 61 106 106 106 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 52 52 52 85 85 85 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 55 55 55 91 91 91 F F F 22 440-459 55 55 55 91 91 91 F F F 23 460-479 60 60 60 105 105 105 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 58 58 58 98 98 98 F F F 26 520-539 59 59 59 101 101 101 F F F 27 540-559 59 59 59 101 101 101 F F F 28 560-579 59 59 59 101 101 101 F F F 29 580-599 60 60 60 102 102 102 F F F Max gen: 61, Max flush: 106 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.448158782Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.492015194Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.49273872Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.492776225Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.492790234Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.492804362Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435} {"msg":"Crucible d05d9621-1829-4182-83fa-ca535ee42ace has session id: 9b4f52e5-6f17-46c9-9ea9-68278945a73e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.493255319Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Upstairs opts: Upstairs UUID: d05d9621-1829-4182-83fa-ca535ee42ace, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.493301004Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Crucible stats registered with UUID: d05d9621-1829-4182-83fa-ca535ee42ace","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.493326031Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523101317Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.52315687Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435} {"msg":"d05d9621-1829-4182-83fa-ca535ee42ace active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523195706Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523226362Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"0","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523274966Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"0","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523311362Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"1","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523345098Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"1","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523376245Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"2","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523409831Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"2","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523440707Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523471754Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.52350365Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60855)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523531167Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57718)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523559013Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"ds_connection connected from Ok(127.0.0.1:32960)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.523786597Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.524164173Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"0","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.524193929Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"1","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.524607951Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"2","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.524904566Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"1","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.524946311Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"0","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525245427Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"2","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.52547099Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525550251Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"[0]R flush_numbers[0..12]: [106, 91, 102, 103, 101, 101, 98, 103, 97, 106, 102, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525803831Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"[0]R generation[0..12]: [61, 55, 60, 60, 59, 59, 58, 60, 58, 61, 60, 61]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525824189Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525840047Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"[1]R flush_numbers[0..12]: [106, 91, 102, 103, 101, 101, 98, 103, 97, 106, 102, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525854025Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"[1]R generation[0..12]: [61, 55, 60, 60, 59, 59, 58, 60, 58, 61, 60, 61]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525867964Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525882082Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"[2]R flush_numbers[0..12]: [106, 91, 102, 103, 101, 101, 98, 103, 97, 106, 102, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525895741Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"[2]R generation[0..12]: [61, 55, 60, 60, 59, 59, 58, 60, 58, 61, 60, 61]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525909279Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525926887Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525940406Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525953784Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.525966942Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:53:57.525979971Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"":"downstairs","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.526015357Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.526029045Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"0","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.526043563Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"1","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.526057022Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"client":"2","":"downstairs","session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.52607056Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.526082989Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"d05d9621-1829-4182-83fa-ca535ee42ace is now active with session: 9b4f52e5-6f17-46c9-9ea9-68278945a73e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.526096097Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.526108066Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435,"session_id":"9b4f52e5-6f17-46c9-9ea9-68278945a73e"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:53:57.526120354Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 459 len 40960 data: 5 5 5 6 7 5 4 5 6 6 02/30 Write block 148 len 32768 data: 6 6 5 6 5 5 5 5 03/30 Read block 415 len 32768 04/30 Read block 530 len 20480 05/30 Read block 18 len 12288 06/30 Read block 555 len 24576 07/30 Write block 289 len 16384 data: 5 5 5 7 08/30 Read block 374 len 28672 09/30 Write block 21 len 4096 data: 5 10/30 Read block 208 len 12288 11/30 Write block 144 len 12288 data: 8 7 5 12/30 Read block 18 len 8192 13/30 Read block 513 len 28672 14/30 Read block 104 len 24576 15/30 Write block 330 len 4096 data: 5 16/30 Write block 415 len 8192 data: 5 5 17/30 Read block 349 len 20480 18/30 Read block 313 len 12288 19/30 Read block 425 len 12288 20/30 Read block 412 len 28672 21/30 Write block 237 len 8192 data: 4 5 22/30 Write block 430 len 12288 data: 2 3 5 23/30 Write block 306 len 32768 data: 4 4 4 3 2 2 2 3 24/30 Read block 68 len 16384 25/30 Read block 172 len 16384 26/30 Read block 148 len 32768 27/30 Write block 302 len 16384 data: 6 6 4 4 28/30 Write block 531 len 40960 data: 9 10 9 10 8 8 9 7 7 7 29/30 Write block 342 len 40960 data: 5 5 5 5 5 4 5 4 5 4 30/30 Write block 329 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:53:57.544724923Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1435} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 106 106 106 F F F 1 020-039 55 55 55 91 91 91 T T T 2 040-059 60 60 60 102 102 102 F F F 3 060-079 60 60 60 103 103 103 F F F 4 080-099 59 59 59 101 101 101 F F F 5 100-119 59 59 59 101 101 101 F F F 6 120-139 58 58 58 98 98 98 F F F 7 140-159 60 60 60 103 103 103 T T T 8 160-179 58 58 58 97 97 97 F F F 9 180-199 61 61 61 106 106 106 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 61 61 61 106 106 106 T T T 12 240-259 61 61 61 106 106 106 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 56 56 56 92 92 92 T T T 15 300-319 59 59 59 101 101 101 T T T 16 320-339 61 61 61 106 106 106 T T T 17 340-359 46 46 46 76 76 76 T T T 18 360-379 52 52 52 85 85 85 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 61 61 61 106 106 106 T T T 21 420-439 55 55 55 91 91 91 T T T 22 440-459 55 55 55 91 91 91 T T T 23 460-479 60 60 60 105 105 105 T T T 24 480-499 58 58 58 98 98 98 F F F 25 500-519 58 58 58 98 98 98 F F F 26 520-539 59 59 59 101 101 101 T T T 27 540-559 59 59 59 101 101 101 T T T 28 560-579 59 59 59 101 101 101 F F F 29 580-599 60 60 60 102 102 102 F F F Max gen: 61, Max flush: 106 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 106 106 106 F F F 1 020-039 62 62 62 107 107 107 F F F 2 040-059 60 60 60 102 102 102 F F F 3 060-079 60 60 60 103 103 103 F F F 4 080-099 59 59 59 101 101 101 F F F 5 100-119 59 59 59 101 101 101 F F F 6 120-139 58 58 58 98 98 98 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 58 58 58 97 97 97 F F F 9 180-199 61 61 61 106 106 106 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 61 61 61 106 106 106 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 62 62 62 107 107 107 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 62 62 62 107 107 107 F F F 17 340-359 62 62 62 107 107 107 F F F 18 360-379 52 52 52 85 85 85 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 62 62 62 107 107 107 F F F 21 420-439 62 62 62 107 107 107 F F F 22 440-459 62 62 62 107 107 107 F F F 23 460-479 62 62 62 107 107 107 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 58 58 58 98 98 98 F F F 26 520-539 62 62 62 107 107 107 F F F 27 540-559 62 62 62 107 107 107 F F F 28 560-579 59 59 59 101 101 101 F F F 29 580-599 60 60 60 102 102 102 F F F Max gen: 62, Max flush: 107 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.861906816Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.906119227Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.907051468Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.907091534Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.907111501Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.90712732Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443} {"msg":"Crucible 305de43b-7d79-4091-878d-36191aeb59be has session id: 13730974-aff3-4537-898c-7e23e11c2c4e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.90755437Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Upstairs opts: Upstairs UUID: 305de43b-7d79-4091-878d-36191aeb59be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.907608633Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Crucible stats registered with UUID: 305de43b-7d79-4091-878d-36191aeb59be","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.90763591Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.93805981Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.938095126Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443} {"msg":"305de43b-7d79-4091-878d-36191aeb59be active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.93814517Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.938166108Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"0","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.938196024Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"0","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.938213882Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"1","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.93823294Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"1","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.938261047Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"2","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.938301672Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"2","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.938335188Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.938358085Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.938375643Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55230)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.938393041Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57314)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.938417988Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56221)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.938452384Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.939113307Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"1","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.939276328Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"0","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.939479205Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"2","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.939719397Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"1","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.939894476Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"0","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940109621Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"2","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940220928Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940406196Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"[0]R flush_numbers[0..12]: [106, 107, 102, 103, 101, 101, 98, 107, 97, 106, 102, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940613772Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"[0]R generation[0..12]: [61, 62, 60, 60, 59, 59, 58, 62, 58, 61, 60, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940637109Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940652688Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"[1]R flush_numbers[0..12]: [106, 107, 102, 103, 101, 101, 98, 107, 97, 106, 102, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940666836Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"[1]R generation[0..12]: [61, 62, 60, 60, 59, 59, 58, 62, 58, 61, 60, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940680544Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940694813Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"[2]R flush_numbers[0..12]: [106, 107, 102, 103, 101, 101, 98, 107, 97, 106, 102, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940708781Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"[2]R generation[0..12]: [61, 62, 60, 60, 59, 59, 58, 62, 58, 61, 60, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940722509Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940736648Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Max found gen is 63","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940749706Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Generation requested: 64 >= found:63","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940763065Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Next flush: 108","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:54:01.940776133Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940796321Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"":"downstairs","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940809739Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940825597Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"0","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940839646Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"1","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940853184Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"client":"2","":"downstairs","session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940866772Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940879151Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"305de43b-7d79-4091-878d-36191aeb59be is now active with session: 13730974-aff3-4537-898c-7e23e11c2c4e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.94089179Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940903948Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443,"session_id":"13730974-aff3-4537-898c-7e23e11c2c4e"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:01.940916217Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 189 len 40960 data: 7 6 7 5 6 7 7 6 6 6 02/30 Read block 221 len 20480 03/30 Flush 04/30 Write block 599 len 4096 data: 3 05/30 Read block 87 len 12288 06/30 Write block 476 len 4096 data: 5 07/30 Flush 08/30 Write block 126 len 40960 data: 7 7 5 4 3 3 3 4 6 6 09/30 Read block 519 len 24576 10/30 Write block 283 len 12288 data: 5 2 2 11/30 Write block 57 len 12288 data: 5 5 5 12/30 Read block 539 len 32768 13/30 Read block 69 len 20480 14/30 Write block 518 len 16384 data: 6 6 4 2 15/30 Read block 230 len 32768 16/30 Write block 342 len 36864 data: 6 6 6 6 6 5 6 5 6 17/30 Read block 377 len 20480 18/30 Write block 290 len 24576 data: 6 6 8 6 5 4 19/30 Read block 364 len 32768 20/30 Read block 559 len 40960 21/30 Read block 218 len 12288 22/30 Write block 358 len 20480 data: 6 7 6 5 5 23/30 Read block 185 len 8192 24/30 Write block 117 len 12288 data: 5 4 5 25/30 Write block 283 len 8192 data: 6 3 26/30 Read block 171 len 16384 27/30 Write block 266 len 20480 data: 10 10 8 7 6 28/30 Read block 112 len 24576 29/30 Read block 561 len 40960 30/30 Read block 310 len 12288 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 10 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 23 23 22 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:01.960912624Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1443} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 106 106 106 F F F 1 020-039 62 62 62 107 107 107 F F F 2 040-059 60 60 60 102 102 102 T T T 3 060-079 60 60 60 103 103 103 F F F 4 080-099 59 59 59 101 101 101 F F F 5 100-119 59 59 59 101 101 101 T T T 6 120-139 58 58 58 98 98 98 T T T 7 140-159 62 62 62 107 107 107 F F F 8 160-179 58 58 58 97 97 97 F F F 9 180-199 64 64 64 108 108 108 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 61 61 61 106 106 106 F F F 13 260-279 59 59 59 101 101 101 T T T 14 280-299 62 62 62 107 107 107 T T T 15 300-319 62 62 62 107 107 107 F F F 16 320-339 62 62 62 107 107 107 F F F 17 340-359 62 62 62 107 107 107 T T T 18 360-379 52 52 52 85 85 85 T T T 19 380-399 61 61 61 106 106 106 F F F 20 400-419 62 62 62 107 107 107 F F F 21 420-439 62 62 62 107 107 107 F F F 22 440-459 62 62 62 107 107 107 F F F 23 460-479 64 64 64 109 109 109 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 58 58 58 98 98 98 T T T 26 520-539 62 62 62 107 107 107 T T T 27 540-559 62 62 62 107 107 107 F F F 28 560-579 59 59 59 101 101 101 F F F 29 580-599 64 64 64 109 109 109 F F F Max gen: 64, Max flush: 109 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 106 106 106 F F F 1 020-039 62 62 62 107 107 107 F F F 2 040-059 65 65 65 110 110 110 F F F 3 060-079 60 60 60 103 103 103 F F F 4 080-099 59 59 59 101 101 101 F F F 5 100-119 65 65 65 110 110 110 F F F 6 120-139 65 65 65 110 110 110 F F F 7 140-159 62 62 62 107 107 107 F F F 8 160-179 58 58 58 97 97 97 F F F 9 180-199 64 64 64 108 108 108 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 61 61 61 106 106 106 F F F 13 260-279 65 65 65 110 110 110 F F F 14 280-299 65 65 65 110 110 110 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 62 62 62 107 107 107 F F F 17 340-359 65 65 65 110 110 110 F F F 18 360-379 65 65 65 110 110 110 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 62 62 62 107 107 107 F F F 21 420-439 62 62 62 107 107 107 F F F 22 440-459 62 62 62 107 107 107 F F F 23 460-479 64 64 64 109 109 109 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 65 65 65 110 110 110 F F F 26 520-539 65 65 65 110 110 110 F F F 27 540-559 62 62 62 107 107 107 F F F 28 560-579 59 59 59 101 101 101 F F F 29 580-599 64 64 64 109 109 109 F F F Max gen: 65, Max flush: 110 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.255506892Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:06.297797188Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.298192522Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.329013965Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.329595717Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.329631223Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.329644892Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.32966093Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451} {"msg":"Crucible 08a4de13-eb4d-468e-b737-7253e54e2583 has session id: fef35ea9-79e0-4caa-af21-97d55d8d1d67","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.330119716Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Upstairs opts: Upstairs UUID: 08a4de13-eb4d-468e-b737-7253e54e2583, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.330165531Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Crucible stats registered with UUID: 08a4de13-eb4d-468e-b737-7253e54e2583","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.330191288Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.360064162Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.360097238Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451} {"msg":"08a4de13-eb4d-468e-b737-7253e54e2583 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.360162251Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.360200756Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"0","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.36025131Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"0","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.360288506Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"1","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.360326121Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"1","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.360387104Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"2","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.36042478Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"2","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.360459446Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.360495992Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.360653633Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"ds_connection connected from Ok(127.0.0.1:36951)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.36068525Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"ds_connection connected from Ok(127.0.0.1:40804)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.360711016Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"ds_connection connected from Ok(127.0.0.1:57501)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.360750722Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.361307007Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"1","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.361347582Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"2","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.36145295Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"0","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.361905187Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"1","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.361981848Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"2","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362034712Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"0","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.36239718Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362503907Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"[0]R flush_numbers[0..12]: [106, 107, 110, 103, 101, 110, 110, 107, 97, 108, 102, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362631792Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"[0]R generation[0..12]: [61, 62, 65, 60, 59, 65, 65, 62, 58, 64, 60, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.36265389Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362669548Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"[1]R flush_numbers[0..12]: [106, 107, 110, 103, 101, 110, 110, 107, 97, 108, 102, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362684186Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"[1]R generation[0..12]: [61, 62, 65, 60, 59, 65, 65, 62, 58, 64, 60, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362697805Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362715883Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"[2]R flush_numbers[0..12]: [106, 107, 110, 103, 101, 110, 110, 107, 97, 108, 102, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362730341Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"[2]R generation[0..12]: [61, 62, 65, 60, 59, 65, 65, 62, 58, 64, 60, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362744079Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362758168Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362771476Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362784755Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Next flush: 111","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:54:06.362797833Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362814921Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"":"downstairs","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362828509Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362840678Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"0","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362854406Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"1","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362867975Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"client":"2","":"downstairs","session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362881663Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362893952Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"08a4de13-eb4d-468e-b737-7253e54e2583 is now active with session: fef35ea9-79e0-4caa-af21-97d55d8d1d67","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.36290672Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362920479Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451,"session_id":"fef35ea9-79e0-4caa-af21-97d55d8d1d67"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:06.362933007Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 454 len 40960 data: 4 5 6 6 6 6 6 6 7 8 02/30 Read block 247 len 4096 03/30 Read block 440 len 4096 04/30 Read block 309 len 12288 05/30 Read block 233 len 8192 06/30 Write block 254 len 8192 data: 7 7 07/30 Write block 169 len 28672 data: 9 9 6 4 3 4 4 08/30 Write block 285 len 40960 data: 3 2 3 5 6 7 7 9 7 6 09/30 Read block 577 len 24576 10/30 Write block 333 len 24576 data: 5 4 3 4 5 5 11/30 Write block 23 len 20480 data: 5 5 5 6 5 12/30 Read block 164 len 16384 13/30 Read block 104 len 16384 14/30 Read block 173 len 32768 15/30 Read block 158 len 32768 16/30 Read block 55 len 24576 17/30 Write block 141 len 16384 data: 8 7 7 9 18/30 Read block 75 len 32768 19/30 Read block 138 len 12288 20/30 Read block 337 len 12288 21/30 Read block 494 len 36864 22/30 Read block 507 len 36864 23/30 Read block 351 len 4096 24/30 Read block 445 len 32768 25/30 Flush 26/30 Write block 530 len 40960 data: 7 10 11 10 11 9 9 10 8 8 27/30 Read block 525 len 24576 28/30 Write block 88 len 28672 data: 8 12 11 13 12 11 9 29/30 Write block 580 len 16384 data: 6 7 6 8 30/30 Read block 444 len 4096 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:06.383681376Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1451} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 106 106 106 F F F 1 020-039 66 66 66 111 111 111 F F F 2 040-059 65 65 65 110 110 110 F F F 3 060-079 60 60 60 103 103 103 F F F 4 080-099 59 59 59 101 101 101 T T T 5 100-119 65 65 65 110 110 110 F F F 6 120-139 65 65 65 110 110 110 F F F 7 140-159 66 66 66 111 111 111 F F F 8 160-179 66 66 66 111 111 111 F F F 9 180-199 64 64 64 108 108 108 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 66 66 66 111 111 111 F F F 13 260-279 65 65 65 110 110 110 F F F 14 280-299 66 66 66 111 111 111 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 66 66 66 111 111 111 F F F 17 340-359 65 65 65 110 110 110 F F F 18 360-379 65 65 65 110 110 110 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 62 62 62 107 107 107 F F F 21 420-439 62 62 62 107 107 107 F F F 22 440-459 66 66 66 111 111 111 F F F 23 460-479 66 66 66 111 111 111 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 65 65 65 110 110 110 F F F 26 520-539 65 65 65 110 110 110 T T T 27 540-559 62 62 62 107 107 107 F F F 28 560-579 59 59 59 101 101 101 F F F 29 580-599 64 64 64 109 109 109 T T T Max gen: 66, Max flush: 111 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 106 106 106 F F F 1 020-039 66 66 66 111 111 111 F F F 2 040-059 65 65 65 110 110 110 F F F 3 060-079 60 60 60 103 103 103 F F F 4 080-099 67 67 67 112 112 112 F F F 5 100-119 65 65 65 110 110 110 F F F 6 120-139 65 65 65 110 110 110 F F F 7 140-159 66 66 66 111 111 111 F F F 8 160-179 66 66 66 111 111 111 F F F 9 180-199 64 64 64 108 108 108 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 62 62 62 107 107 107 F F F 12 240-259 66 66 66 111 111 111 F F F 13 260-279 65 65 65 110 110 110 F F F 14 280-299 66 66 66 111 111 111 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 66 66 66 111 111 111 F F F 17 340-359 65 65 65 110 110 110 F F F 18 360-379 65 65 65 110 110 110 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 62 62 62 107 107 107 F F F 21 420-439 62 62 62 107 107 107 F F F 22 440-459 66 66 66 111 111 111 F F F 23 460-479 66 66 66 111 111 111 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 65 65 65 110 110 110 F F F 26 520-539 67 67 67 112 112 112 F F F 27 540-559 62 62 62 107 107 107 F F F 28 560-579 59 59 59 101 101 101 F F F 29 580-599 67 67 67 112 112 112 F F F Max gen: 67, Max flush: 112 Begin loop 34 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.675039943Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:10.718019498Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.718494452Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.749670495Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.750388251Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.750431956Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.750454913Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.750473361Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459} {"msg":"Crucible 3d65e65f-6ab5-4dde-b500-8404708c3b52 has session id: edab2d5e-373a-40b2-bbdd-bf41b87bc0f4","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.750901891Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"Upstairs opts: Upstairs UUID: 3d65e65f-6ab5-4dde-b500-8404708c3b52, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.750959664Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"Crucible stats registered with UUID: 3d65e65f-6ab5-4dde-b500-8404708c3b52","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.750984631Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.78150503Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.781545045Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459} {"msg":"3d65e65f-6ab5-4dde-b500-8404708c3b52 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.781570892Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.78158775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"0","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.781616187Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"0","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.781635374Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"1","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.781653352Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"1","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.78167013Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"2","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.781688068Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"2","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.781710496Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.781728424Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.781745552Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43555)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.781852989Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43889)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.781939729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:62580)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.781984294Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.782693651Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"2","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.782883959Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"1","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.783148288Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"0","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.783364973Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"2","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.783533563Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"1","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.783921558Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.783956964Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"0","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.78415863Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"[0]R flush_numbers[0..12]: [106, 111, 110, 103, 112, 110, 110, 111, 111, 108, 102, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784492661Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"[0]R generation[0..12]: [61, 66, 65, 60, 67, 65, 65, 66, 66, 64, 60, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784534326Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784561633Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"[1]R flush_numbers[0..12]: [106, 111, 110, 103, 112, 110, 110, 111, 111, 108, 102, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.78458522Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"[1]R generation[0..12]: [61, 66, 65, 60, 67, 65, 65, 66, 66, 64, 60, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784609387Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784635775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"[2]R flush_numbers[0..12]: [106, 111, 110, 103, 112, 110, 110, 111, 111, 108, 102, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784662521Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"[2]R generation[0..12]: [61, 66, 65, 60, 67, 65, 65, 66, 66, 64, 60, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784686918Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784716505Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784749431Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784778998Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784807024Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784834431Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"":"downstairs","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784862368Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784888275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"0","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784920451Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"1","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784943728Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"client":"2","":"downstairs","session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784958577Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784970865Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"3d65e65f-6ab5-4dde-b500-8404708c3b52 is now active with session: edab2d5e-373a-40b2-bbdd-bf41b87bc0f4","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.784988273Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.785000552Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459,"session_id":"edab2d5e-373a-40b2-bbdd-bf41b87bc0f4"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:10.785013011Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 168 len 28672 data: 11 10 10 7 5 4 5 02/30 Read block 30 len 36864 03/30 Write block 559 len 16384 data: 5 4 4 4 04/30 Write block 272 len 8192 data: 5 3 05/30 Write block 285 len 12288 data: 4 3 4 06/30 Read block 202 len 12288 07/30 Write block 399 len 8192 data: 6 5 08/30 Read block 59 len 12288 09/30 Flush 10/30 Write block 21 len 20480 data: 6 6 6 6 6 11/30 Read block 590 len 12288 12/30 Write block 500 len 20480 data: 9 9 8 8 7 13/30 Write block 437 len 32768 data: 6 6 4 4 4 4 5 5 14/30 Read block 405 len 12288 15/30 Read block 477 len 4096 16/30 Read block 590 len 16384 17/30 Write block 262 len 20480 data: 7 7 8 9 11 18/30 Read block 169 len 16384 19/30 Read block 586 len 32768 20/30 Write block 253 len 24576 data: 7 8 8 5 5 5 21/30 Write block 43 len 24576 data: 10 9 8 9 7 7 22/30 Write block 519 len 24576 data: 7 5 3 3 3 3 23/30 Read block 419 len 8192 24/30 Write block 336 len 40960 data: 5 6 6 5 4 5 7 7 7 7 25/30 Flush 26/30 Write block 561 len 40960 data: 5 5 3 4 4 4 5 5 7 6 27/30 Write block 228 len 20480 data: 6 6 7 6 7 28/30 Read block 153 len 20480 29/30 Read block 49 len 36864 30/30 Write block 123 len 24576 data: 7 6 6 8 8 6 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:10.804848466Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1459} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 106 106 106 F F F 1 020-039 68 68 68 114 114 114 F F F 2 040-059 68 68 68 114 114 114 F F F 3 060-079 60 60 60 103 103 103 F F F 4 080-099 67 67 67 112 112 112 F F F 5 100-119 65 65 65 110 110 110 F F F 6 120-139 65 65 65 110 110 110 T T T 7 140-159 66 66 66 111 111 111 F F F 8 160-179 68 68 68 113 113 113 F F F 9 180-199 64 64 64 108 108 108 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 62 62 62 107 107 107 T T T 12 240-259 68 68 68 114 114 114 F F F 13 260-279 68 68 68 114 114 114 F F F 14 280-299 68 68 68 113 113 113 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 68 68 68 114 114 114 F F F 17 340-359 68 68 68 114 114 114 F F F 18 360-379 65 65 65 110 110 110 F F F 19 380-399 68 68 68 113 113 113 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 68 68 68 114 114 114 F F F 22 440-459 68 68 68 114 114 114 F F F 23 460-479 66 66 66 111 111 111 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 68 68 68 114 114 114 F F F 26 520-539 68 68 68 114 114 114 F F F 27 540-559 68 68 68 113 113 113 F F F 28 560-579 68 68 68 113 113 113 T T T 29 580-599 67 67 67 112 112 112 F F F Max gen: 68, Max flush: 114 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 106 106 106 F F F 1 020-039 68 68 68 114 114 114 F F F 2 040-059 68 68 68 114 114 114 F F F 3 060-079 60 60 60 103 103 103 F F F 4 080-099 67 67 67 112 112 112 F F F 5 100-119 65 65 65 110 110 110 F F F 6 120-139 69 69 69 115 115 115 F F F 7 140-159 66 66 66 111 111 111 F F F 8 160-179 68 68 68 113 113 113 F F F 9 180-199 64 64 64 108 108 108 F F F 10 200-219 60 60 60 102 102 102 F F F 11 220-239 69 69 69 115 115 115 F F F 12 240-259 68 68 68 114 114 114 F F F 13 260-279 68 68 68 114 114 114 F F F 14 280-299 68 68 68 113 113 113 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 68 68 68 114 114 114 F F F 17 340-359 68 68 68 114 114 114 F F F 18 360-379 65 65 65 110 110 110 F F F 19 380-399 68 68 68 113 113 113 F F F 20 400-419 68 68 68 113 113 113 F F F 21 420-439 68 68 68 114 114 114 F F F 22 440-459 68 68 68 114 114 114 F F F 23 460-479 66 66 66 111 111 111 F F F 24 480-499 58 58 58 98 98 98 F F F 25 500-519 68 68 68 114 114 114 F F F 26 520-539 68 68 68 114 114 114 F F F 27 540-559 68 68 68 113 113 113 F F F 28 560-579 69 69 69 115 115 115 F F F 29 580-599 67 67 67 112 112 112 F F F Max gen: 69, Max flush: 115 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.103245761Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.148687429Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.149411045Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.149458629Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.149476487Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.149494255Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467} {"msg":"Crucible bd04194c-7a7e-41dc-a0fe-d02c03773b24 has session id: ba4016e9-8e4c-4de3-ae21-46c3ec91267d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.149860252Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Upstairs opts: Upstairs UUID: bd04194c-7a7e-41dc-a0fe-d02c03773b24, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.149921535Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Crucible stats registered with UUID: bd04194c-7a7e-41dc-a0fe-d02c03773b24","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.149947072Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.179858932Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.179903356Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467} {"msg":"bd04194c-7a7e-41dc-a0fe-d02c03773b24 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.179927953Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.179944632Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"0","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.179974098Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"0","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.179998455Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"1","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.180017713Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"1","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.180034231Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"2","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.180051699Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"2","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.180068137Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.180085235Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.180163406Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63097)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.180194182Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48112)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.180222719Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58503)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.180322738Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.181052692Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"0","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.181087608Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"2","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.181283465Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"1","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.181827162Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"2","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.18201206Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"1","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182053436Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"0","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182395166Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182572825Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"[0]R flush_numbers[0..12]: [106, 114, 114, 103, 112, 110, 115, 111, 113, 108, 102, 115]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182682702Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"[0]R generation[0..12]: [61, 68, 68, 60, 67, 65, 69, 66, 68, 64, 60, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182713939Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182731466Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"[1]R flush_numbers[0..12]: [106, 114, 114, 103, 112, 110, 115, 111, 113, 108, 102, 115]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182746985Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"[1]R generation[0..12]: [61, 68, 68, 60, 67, 65, 69, 66, 68, 64, 60, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182760833Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182775011Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"[2]R flush_numbers[0..12]: [106, 114, 114, 103, 112, 110, 115, 111, 113, 108, 102, 115]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.18278901Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"[2]R generation[0..12]: [61, 68, 68, 60, 67, 65, 69, 66, 68, 64, 60, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182802798Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182816907Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182831035Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182848143Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182861891Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182888238Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"":"downstairs","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182902126Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182915065Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"0","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182928853Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"1","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182942242Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"client":"2","":"downstairs","session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.18295565Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182968109Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"bd04194c-7a7e-41dc-a0fe-d02c03773b24 is now active with session: ba4016e9-8e4c-4de3-ae21-46c3ec91267d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182980987Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.182993066Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467,"session_id":"ba4016e9-8e4c-4de3-ae21-46c3ec91267d"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:15.183005405Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 199 len 36864 data: 6 6 6 5 4 5 5 5 6 02/30 Read block 514 len 24576 03/30 Read block 436 len 24576 04/30 Read block 217 len 20480 05/30 Read block 260 len 12288 06/30 Write block 409 len 12288 data: 7 8 6 07/30 Read block 91 len 24576 08/30 Write block 93 len 4096 data: 12 09/30 Read block 237 len 24576 10/30 Read block 1 len 20480 11/30 Write block 82 len 36864 data: 6 6 5 7 8 9 9 13 12 12/30 Write block 230 len 40960 data: 8 7 8 5 4 5 4 5 6 4 13/30 Flush 14/30 Write block 434 len 28672 data: 4 4 6 7 7 5 5 15/30 Read block 522 len 8192 16/30 Read block 424 len 28672 17/30 Write block 66 len 24576 data: 6 7 7 6 6 6 18/30 Read block 24 len 12288 19/30 Write block 203 len 40960 data: 5 6 6 6 7 6 7 7 8 9 20/30 Write block 412 len 20480 data: 7 8 7 6 6 21/30 Read block 13 len 40960 22/30 Write block 34 len 36864 data: 2 2 3 3 3 4 3 6 6 23/30 Read block 512 len 32768 24/30 Write block 397 len 28672 data: 7 6 7 6 5 6 6 25/30 Write block 160 len 16384 data: 9 10 11 9 26/30 Write block 478 len 16384 data: 7 7 6 5 27/30 Write block 566 len 28672 data: 5 6 6 8 7 5 8 28/30 Write block 48 len 32768 data: 8 8 9 8 9 8 8 8 29/30 Write block 203 len 28672 data: 6 7 7 7 8 7 8 30/30 Write block 291 len 36864 data: 8 10 8 7 5 3 5 5 5 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 7 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 7 Done Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 16 15 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:15.204099383Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1467} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 106 106 106 F F F 1 020-039 68 68 68 114 114 114 T T T 2 040-059 68 68 68 114 114 114 T T T 3 060-079 60 60 60 103 103 103 T T T 4 080-099 70 70 70 116 116 116 F F F 5 100-119 65 65 65 110 110 110 F F F 6 120-139 69 69 69 115 115 115 F F F 7 140-159 66 66 66 111 111 111 F F F 8 160-179 68 68 68 113 113 113 T T T 9 180-199 70 70 70 116 116 116 F F F 10 200-219 70 70 70 116 116 116 T T T 11 220-239 70 70 70 116 116 116 F F F 12 240-259 68 68 68 114 114 114 F F F 13 260-279 68 68 68 114 114 114 F F F 14 280-299 68 68 68 113 113 113 T T T 15 300-319 62 62 62 107 107 107 F F F 16 320-339 68 68 68 114 114 114 F F F 17 340-359 68 68 68 114 114 114 F F F 18 360-379 65 65 65 110 110 110 F F F 19 380-399 68 68 68 113 113 113 T T T 20 400-419 70 70 70 116 116 116 T T T 21 420-439 68 68 68 114 114 114 T T T 22 440-459 68 68 68 114 114 114 T T T 23 460-479 66 66 66 111 111 111 T T T 24 480-499 58 58 58 98 98 98 T T T 25 500-519 68 68 68 114 114 114 F F F 26 520-539 68 68 68 114 114 114 F F F 27 540-559 68 68 68 113 113 113 F F F 28 560-579 69 69 69 115 115 115 T T T 29 580-599 67 67 67 112 112 112 F F F Max gen: 70, Max flush: 116 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 106 106 106 F F F 1 020-039 71 71 71 117 117 117 F F F 2 040-059 71 71 71 117 117 117 F F F 3 060-079 71 71 71 117 117 117 F F F 4 080-099 70 70 70 116 116 116 F F F 5 100-119 65 65 65 110 110 110 F F F 6 120-139 69 69 69 115 115 115 F F F 7 140-159 66 66 66 111 111 111 F F F 8 160-179 71 71 71 117 117 117 F F F 9 180-199 70 70 70 116 116 116 F F F 10 200-219 71 71 71 117 117 117 F F F 11 220-239 70 70 70 116 116 116 F F F 12 240-259 68 68 68 114 114 114 F F F 13 260-279 68 68 68 114 114 114 F F F 14 280-299 71 71 71 117 117 117 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 68 68 68 114 114 114 F F F 17 340-359 68 68 68 114 114 114 F F F 18 360-379 65 65 65 110 110 110 F F F 19 380-399 71 71 71 117 117 117 F F F 20 400-419 71 71 71 117 117 117 F F F 21 420-439 71 71 71 117 117 117 F F F 22 440-459 71 71 71 117 117 117 F F F 23 460-479 71 71 71 117 117 117 F F F 24 480-499 71 71 71 117 117 117 F F F 25 500-519 68 68 68 114 114 114 F F F 26 520-539 68 68 68 114 114 114 F F F 27 540-559 68 68 68 113 113 113 F F F 28 560-579 71 71 71 117 117 117 F F F 29 580-599 67 67 67 112 112 112 F F F Max gen: 71, Max flush: 117 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.529543733Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:19.574709723Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.575287755Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.608367366Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.609302616Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.609352471Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.609370858Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.609389746Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475} {"msg":"Crucible 2d0c600a-667d-4bd5-9ae2-2b10e333445f has session id: 31de95a4-d1dc-4621-9d57-e2f37f58f3b7","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.609858801Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Upstairs opts: Upstairs UUID: 2d0c600a-667d-4bd5-9ae2-2b10e333445f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.609908955Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Crucible stats registered with UUID: 2d0c600a-667d-4bd5-9ae2-2b10e333445f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.609934712Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.64012327Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640180163Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475} {"msg":"2d0c600a-667d-4bd5-9ae2-2b10e333445f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640197971Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640214999Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"0","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640246636Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"0","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640264804Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"1","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640282542Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"1","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640298749Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"2","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640316018Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"2","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640332386Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640349474Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640372451Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:55014)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640582576Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62331)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640646159Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:39152)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.640682695Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.641378254Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"2","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.641573041Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"1","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.641696146Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"0","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.642173891Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"2","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.642218056Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"1","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.642302336Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"0","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.642761562Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.64286884Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"[0]R flush_numbers[0..12]: [106, 117, 117, 117, 116, 110, 115, 111, 117, 116, 117, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.642998654Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"[0]R generation[0..12]: [61, 71, 71, 71, 70, 65, 69, 66, 71, 70, 71, 70]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643021432Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643037Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"[1]R flush_numbers[0..12]: [106, 117, 117, 117, 116, 110, 115, 111, 117, 116, 117, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643055978Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"[1]R generation[0..12]: [61, 71, 71, 71, 70, 65, 69, 66, 71, 70, 71, 70]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643070906Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643086104Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"[2]R flush_numbers[0..12]: [106, 117, 117, 117, 116, 110, 115, 111, 117, 116, 117, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643100293Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"[2]R generation[0..12]: [61, 71, 71, 71, 70, 65, 69, 66, 71, 70, 71, 70]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643114021Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643128159Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643141288Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643154656Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643168655Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:54:19.643181703Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"":"downstairs","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643199891Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643212069Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"0","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643225728Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"1","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643239556Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"client":"2","":"downstairs","session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643255935Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643268743Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"2d0c600a-667d-4bd5-9ae2-2b10e333445f is now active with session: 31de95a4-d1dc-4621-9d57-e2f37f58f3b7","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643281852Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.64329405Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475,"session_id":"31de95a4-d1dc-4621-9d57-e2f37f58f3b7"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:19.643306698Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 222 len 16384 data: 5 5 5 6 02/30 Read block 89 len 12288 03/30 Write block 495 len 40960 data: 3 3 3 4 7 10 10 9 9 8 04/30 Read block 490 len 24576 05/30 Write block 274 len 12288 data: 6 6 7 06/30 Write block 123 len 28672 data: 8 7 7 9 9 7 5 07/30 Write block 117 len 36864 data: 6 5 6 5 5 6 9 8 8 08/30 Write block 416 len 24576 data: 7 5 4 4 4 5 09/30 Write block 12 len 32768 data: 4 5 5 3 6 6 8 8 10/30 Write block 82 len 32768 data: 7 7 6 8 9 10 10 14 11/30 Read block 2 len 28672 12/30 Write block 362 len 24576 data: 6 6 5 5 5 6 13/30 Write block 90 len 32768 data: 13 14 13 13 10 9 8 7 14/30 Write block 7 len 24576 data: 8 7 5 4 4 5 15/30 Read block 388 len 4096 16/30 Write block 218 len 28672 data: 9 7 7 7 6 6 6 17/30 Read block 418 len 36864 18/30 Write block 38 len 28672 data: 4 5 4 7 7 11 10 19/30 Write block 286 len 28672 data: 4 5 6 7 8 9 11 20/30 Read block 522 len 28672 21/30 Read block 205 len 8192 22/30 Read block 344 len 32768 23/30 Write block 590 len 28672 data: 8 10 9 7 7 5 5 24/30 Write block 410 len 16384 data: 9 7 8 9 25/30 Read block 0 len 28672 26/30 Write block 510 len 28672 data: 5 5 5 4 5 5 8 27/30 Write block 541 len 24576 data: 6 4 5 5 6 7 28/30 Read block 334 len 4096 29/30 Write block 593 len 24576 data: 8 8 6 6 4 4 30/30 Write block 257 len 32768 data: 6 6 6 8 8 8 8 9 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 6 Sent Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 28 28 29 85 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:19.663131375Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1475} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 106 106 106 T T T 1 020-039 71 71 71 117 117 117 T T T 2 040-059 71 71 71 117 117 117 T T T 3 060-079 71 71 71 117 117 117 F F F 4 080-099 70 70 70 116 116 116 T T T 5 100-119 65 65 65 110 110 110 T T T 6 120-139 69 69 69 115 115 115 T T T 7 140-159 66 66 66 111 111 111 F F F 8 160-179 71 71 71 117 117 117 F F F 9 180-199 70 70 70 116 116 116 F F F 10 200-219 71 71 71 117 117 117 T T T 11 220-239 70 70 70 116 116 116 T T T 12 240-259 68 68 68 114 114 114 T T T 13 260-279 68 68 68 114 114 114 T T T 14 280-299 71 71 71 117 117 117 T T T 15 300-319 62 62 62 107 107 107 F F F 16 320-339 68 68 68 114 114 114 F F F 17 340-359 68 68 68 114 114 114 F F F 18 360-379 65 65 65 110 110 110 T T T 19 380-399 71 71 71 117 117 117 F F F 20 400-419 71 71 71 117 117 117 T T T 21 420-439 71 71 71 117 117 117 T T T 22 440-459 71 71 71 117 117 117 F F F 23 460-479 71 71 71 117 117 117 F F F 24 480-499 71 71 71 117 117 117 T T T 25 500-519 68 68 68 114 114 114 T T T 26 520-539 68 68 68 114 114 114 F F F 27 540-559 68 68 68 113 113 113 T T T 28 560-579 71 71 71 117 117 117 F F F 29 580-599 67 67 67 112 112 112 T T T Max gen: 71, Max flush: 117 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 118 118 118 F F F 1 020-039 72 72 72 118 118 118 F F F 2 040-059 72 72 72 118 118 118 F F F 3 060-079 71 71 71 117 117 117 F F F 4 080-099 72 72 72 118 118 118 F F F 5 100-119 72 72 72 118 118 118 F F F 6 120-139 72 72 72 118 118 118 F F F 7 140-159 66 66 66 111 111 111 F F F 8 160-179 71 71 71 117 117 117 F F F 9 180-199 70 70 70 116 116 116 F F F 10 200-219 72 72 72 118 118 118 F F F 11 220-239 72 72 72 118 118 118 F F F 12 240-259 72 72 72 118 118 118 F F F 13 260-279 72 72 72 118 118 118 F F F 14 280-299 72 72 72 118 118 118 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 68 68 68 114 114 114 F F F 17 340-359 68 68 68 114 114 114 F F F 18 360-379 72 72 72 118 118 118 F F F 19 380-399 71 71 71 117 117 117 F F F 20 400-419 72 72 72 118 118 118 F F F 21 420-439 72 72 72 118 118 118 F F F 22 440-459 71 71 71 117 117 117 F F F 23 460-479 71 71 71 117 117 117 F F F 24 480-499 72 72 72 118 118 118 F F F 25 500-519 72 72 72 118 118 118 F F F 26 520-539 68 68 68 114 114 114 F F F 27 540-559 72 72 72 118 118 118 F F F 28 560-579 71 71 71 117 117 117 F F F 29 580-599 72 72 72 118 118 118 F F F Max gen: 72, Max flush: 118 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.046641887Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.090598338Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.091285458Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.091323444Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.091337462Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.091351801Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483} {"msg":"Crucible 252a7f89-4209-45c0-b078-7fd8ea968f51 has session id: fce19c97-5cd3-451e-b108-de7f55368900","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.091826805Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Upstairs opts: Upstairs UUID: 252a7f89-4209-45c0-b078-7fd8ea968f51, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.09187446Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Crucible stats registered with UUID: 252a7f89-4209-45c0-b078-7fd8ea968f51","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.091899837Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.121794198Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.121833054Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483} {"msg":"252a7f89-4209-45c0-b078-7fd8ea968f51 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.12186698Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.121883928Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"0","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.121913255Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"0","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.121931522Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"1","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.12194968Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"1","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.121966608Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"2","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.121988726Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"2","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.122052798Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.122088244Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.12212307Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"ds_connection connected from Ok(127.0.0.1:65302)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.122154846Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"ds_connection connected from Ok(127.0.0.1:42273)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.122188532Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"ds_connection connected from Ok(127.0.0.1:41873)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.12229412Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.122943884Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"2","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.12297766Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"0","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.123010756Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"1","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.123596548Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"0","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.123768088Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"1","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.123803854Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"2","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124142914Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.12434811Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"[0]R flush_numbers[0..12]: [118, 118, 118, 117, 118, 118, 118, 111, 117, 116, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124472366Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"[0]R generation[0..12]: [72, 72, 72, 71, 72, 72, 72, 66, 71, 70, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124493094Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124508212Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"[1]R flush_numbers[0..12]: [118, 118, 118, 117, 118, 118, 118, 111, 117, 116, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.12452242Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"[1]R generation[0..12]: [72, 72, 72, 71, 72, 72, 72, 66, 71, 70, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124536278Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124550207Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"[2]R flush_numbers[0..12]: [118, 118, 118, 117, 118, 118, 118, 111, 117, 116, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124563985Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"[2]R generation[0..12]: [72, 72, 72, 71, 72, 72, 72, 66, 71, 70, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124581833Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124596311Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Max found gen is 73","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.12460944Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Generation requested: 74 >= found:73","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124622648Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124635667Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124655395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"":"downstairs","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124668773Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124680821Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"0","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.12469411Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"1","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124706989Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"client":"2","":"downstairs","session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124720237Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124732775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"252a7f89-4209-45c0-b078-7fd8ea968f51 is now active with session: fce19c97-5cd3-451e-b108-de7f55368900","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124745414Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124757352Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483,"session_id":"fce19c97-5cd3-451e-b108-de7f55368900"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:24.124769581Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 537 len 24576 data: 11 9 9 8 7 5 02/30 Read block 493 len 12288 03/30 Write block 515 len 8192 data: 6 9 04/30 Write block 354 len 4096 data: 5 05/30 Write block 188 len 16384 data: 6 8 7 8 06/30 Write block 4 len 4096 data: 4 07/30 Read block 573 len 24576 08/30 Write block 28 len 12288 data: 5 5 4 09/30 Read block 353 len 8192 10/30 Read block 46 len 28672 11/30 Read block 139 len 12288 12/30 Read block 327 len 32768 13/30 Read block 205 len 8192 14/30 Write block 66 len 20480 data: 7 8 8 7 7 15/30 Flush 16/30 Write block 499 len 8192 data: 8 11 17/30 Flush 18/30 Write block 38 len 32768 data: 5 6 5 8 8 12 11 9 19/30 Read block 304 len 20480 20/30 Write block 503 len 12288 data: 10 9 7 21/30 Write block 135 len 40960 data: 7 7 7 6 6 8 9 8 8 10 22/30 Read block 308 len 12288 23/30 Read block 17 len 20480 24/30 Read block 182 len 40960 25/30 Write block 323 len 36864 data: 6 5 7 5 4 5 6 6 5 26/30 Read block 73 len 40960 27/30 Read block 425 len 32768 28/30 Write block 419 len 8192 data: 5 5 29/30 Read block 560 len 36864 30/30 Read block 317 len 4096 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 13 13 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:24.14414899Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1483} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 119 119 119 F F F 1 020-039 74 74 74 119 119 119 T T T 2 040-059 72 72 72 118 118 118 T T T 3 060-079 74 74 74 119 119 119 F F F 4 080-099 72 72 72 118 118 118 F F F 5 100-119 72 72 72 118 118 118 F F F 6 120-139 72 72 72 118 118 118 T T T 7 140-159 66 66 66 111 111 111 T T T 8 160-179 71 71 71 117 117 117 F F F 9 180-199 74 74 74 119 119 119 F F F 10 200-219 72 72 72 118 118 118 F F F 11 220-239 72 72 72 118 118 118 F F F 12 240-259 72 72 72 118 118 118 F F F 13 260-279 72 72 72 118 118 118 F F F 14 280-299 72 72 72 118 118 118 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 68 68 68 114 114 114 T T T 17 340-359 74 74 74 119 119 119 F F F 18 360-379 72 72 72 118 118 118 F F F 19 380-399 71 71 71 117 117 117 F F F 20 400-419 72 72 72 118 118 118 T T T 21 420-439 72 72 72 118 118 118 T T T 22 440-459 71 71 71 117 117 117 F F F 23 460-479 71 71 71 117 117 117 F F F 24 480-499 74 74 74 120 120 120 F F F 25 500-519 74 74 74 120 120 120 T T T 26 520-539 74 74 74 119 119 119 F F F 27 540-559 74 74 74 119 119 119 F F F 28 560-579 71 71 71 117 117 117 F F F 29 580-599 72 72 72 118 118 118 F F F Max gen: 74, Max flush: 120 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 119 119 119 F F F 1 020-039 75 75 75 121 121 121 F F F 2 040-059 75 75 75 121 121 121 F F F 3 060-079 74 74 74 119 119 119 F F F 4 080-099 72 72 72 118 118 118 F F F 5 100-119 72 72 72 118 118 118 F F F 6 120-139 75 75 75 121 121 121 F F F 7 140-159 75 75 75 121 121 121 F F F 8 160-179 71 71 71 117 117 117 F F F 9 180-199 74 74 74 119 119 119 F F F 10 200-219 72 72 72 118 118 118 F F F 11 220-239 72 72 72 118 118 118 F F F 12 240-259 72 72 72 118 118 118 F F F 13 260-279 72 72 72 118 118 118 F F F 14 280-299 72 72 72 118 118 118 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 75 75 75 121 121 121 F F F 17 340-359 74 74 74 119 119 119 F F F 18 360-379 72 72 72 118 118 118 F F F 19 380-399 71 71 71 117 117 117 F F F 20 400-419 75 75 75 121 121 121 F F F 21 420-439 75 75 75 121 121 121 F F F 22 440-459 71 71 71 117 117 117 F F F 23 460-479 71 71 71 117 117 117 F F F 24 480-499 74 74 74 120 120 120 F F F 25 500-519 75 75 75 121 121 121 F F F 26 520-539 74 74 74 119 119 119 F F F 27 540-559 74 74 74 119 119 119 F F F 28 560-579 71 71 71 117 117 117 F F F 29 580-599 72 72 72 118 118 118 F F F Max gen: 75, Max flush: 121 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.435798612Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.482536118Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.483516254Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.483556489Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.483570957Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.483585136Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491} {"msg":"Crucible 1d81a176-440d-4926-a396-685cde758969 has session id: ae36938e-f080-475c-b791-62e734250b91","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.484190775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Upstairs opts: Upstairs UUID: 1d81a176-440d-4926-a396-685cde758969, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.484239079Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Crucible stats registered with UUID: 1d81a176-440d-4926-a396-685cde758969","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.484263687Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.51499941Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.515046275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491} {"msg":"1d81a176-440d-4926-a396-685cde758969 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.515076011Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.51508923Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"0","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.515114857Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"0","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.515129155Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"1","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.515143663Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"1","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.515156832Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"2","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.51517154Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"2","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.515220144Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.515242992Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.515272158Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"ds_connection connected from Ok(127.0.0.1:51753)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.515475615Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"ds_connection connected from Ok(127.0.0.1:37941)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.515531108Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"ds_connection connected from Ok(127.0.0.1:35114)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.515557015Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.51636811Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"0","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.516402706Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"2","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.5165465Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"1","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.517089726Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"2","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.517120523Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"0","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.517251198Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"1","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.517673238Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.517766897Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"[0]R flush_numbers[0..12]: [119, 121, 121, 119, 118, 118, 121, 121, 117, 119, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.517894692Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"[0]R generation[0..12]: [74, 75, 75, 74, 72, 72, 75, 75, 71, 74, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.5179145Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.517929288Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"[1]R flush_numbers[0..12]: [119, 121, 121, 119, 118, 118, 121, 121, 117, 119, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.517946996Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"[1]R generation[0..12]: [74, 75, 75, 74, 72, 72, 75, 75, 71, 74, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.517961315Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.517975373Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"[2]R flush_numbers[0..12]: [119, 121, 121, 119, 118, 118, 121, 121, 117, 119, 118, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.517989261Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"[2]R generation[0..12]: [74, 75, 75, 74, 72, 72, 75, 75, 71, 74, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.518014538Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.518031656Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.518044715Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:54:28.518059123Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Next flush: 122","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.518076471Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.51808946Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"":"downstairs","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.518102788Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.518114707Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"0","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.518127955Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"1","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.518141034Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"client":"2","":"downstairs","session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.518157382Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.51817005Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"1d81a176-440d-4926-a396-685cde758969 is now active with session: ae36938e-f080-475c-b791-62e734250b91","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.518182949Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.518195317Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491,"session_id":"ae36938e-f080-475c-b791-62e734250b91"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:28.518207736Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 443 len 36864 data: 6 6 4 4 3 3 5 5 5 02/30 Read block 502 len 20480 03/30 Flush 04/30 Write block 501 len 20480 data: 11 10 11 10 8 05/30 Read block 312 len 28672 06/30 Write block 589 len 20480 data: 9 9 11 10 9 07/30 Flush 08/30 Write block 164 len 40960 data: 9 7 9 10 12 11 11 8 6 5 09/30 Read block 514 len 8192 10/30 Read block 403 len 28672 11/30 Read block 514 len 24576 12/30 Read block 497 len 40960 13/30 Read block 399 len 20480 14/30 Read block 9 len 32768 15/30 Read block 72 len 28672 16/30 Read block 271 len 32768 17/30 Write block 506 len 8192 data: 7 6 18/30 Read block 38 len 16384 19/30 Write block 358 len 8192 data: 7 8 20/30 Write block 571 len 36864 data: 6 9 7 6 7 6 6 6 4 21/30 Write block 559 len 32768 data: 6 5 6 6 4 5 5 6 22/30 Write block 158 len 40960 data: 8 8 10 11 12 10 10 8 10 11 23/30 Flush 24/30 Write block 227 len 20480 data: 4 7 7 9 8 25/30 Read block 57 len 16384 26/30 Read block 339 len 24576 27/30 Read block 414 len 20480 28/30 Write block 382 len 20480 data: 7 7 7 6 6 29/30 Write block 364 len 20480 data: 6 6 6 7 6 30/30 Read block 539 len 40960 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:28.540159014Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1491} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 119 119 119 F F F 1 020-039 75 75 75 121 121 121 F F F 2 040-059 75 75 75 121 121 121 F F F 3 060-079 74 74 74 119 119 119 F F F 4 080-099 72 72 72 118 118 118 F F F 5 100-119 72 72 72 118 118 118 F F F 6 120-139 75 75 75 121 121 121 F F F 7 140-159 76 76 76 124 124 124 F F F 8 160-179 76 76 76 124 124 124 F F F 9 180-199 74 74 74 119 119 119 F F F 10 200-219 72 72 72 118 118 118 F F F 11 220-239 72 72 72 118 118 118 T T T 12 240-259 72 72 72 118 118 118 F F F 13 260-279 72 72 72 118 118 118 F F F 14 280-299 72 72 72 118 118 118 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 75 75 75 121 121 121 F F F 17 340-359 76 76 76 124 124 124 F F F 18 360-379 72 72 72 118 118 118 T T T 19 380-399 71 71 71 117 117 117 T T T 20 400-419 75 75 75 121 121 121 F F F 21 420-439 75 75 75 121 121 121 F F F 22 440-459 76 76 76 122 122 122 F F F 23 460-479 71 71 71 117 117 117 F F F 24 480-499 74 74 74 120 120 120 F F F 25 500-519 76 76 76 124 124 124 F F F 26 520-539 74 74 74 119 119 119 F F F 27 540-559 76 76 76 124 124 124 F F F 28 560-579 76 76 76 124 124 124 F F F 29 580-599 76 76 76 123 123 123 F F F Max gen: 76, Max flush: 124 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 119 119 119 F F F 1 020-039 75 75 75 121 121 121 F F F 2 040-059 75 75 75 121 121 121 F F F 3 060-079 74 74 74 119 119 119 F F F 4 080-099 72 72 72 118 118 118 F F F 5 100-119 72 72 72 118 118 118 F F F 6 120-139 75 75 75 121 121 121 F F F 7 140-159 76 76 76 124 124 124 F F F 8 160-179 76 76 76 124 124 124 F F F 9 180-199 74 74 74 119 119 119 F F F 10 200-219 72 72 72 118 118 118 F F F 11 220-239 77 77 77 125 125 125 F F F 12 240-259 72 72 72 118 118 118 F F F 13 260-279 72 72 72 118 118 118 F F F 14 280-299 72 72 72 118 118 118 F F F 15 300-319 62 62 62 107 107 107 F F F 16 320-339 75 75 75 121 121 121 F F F 17 340-359 76 76 76 124 124 124 F F F 18 360-379 77 77 77 125 125 125 F F F 19 380-399 77 77 77 125 125 125 F F F 20 400-419 75 75 75 121 121 121 F F F 21 420-439 75 75 75 121 121 121 F F F 22 440-459 76 76 76 122 122 122 F F F 23 460-479 71 71 71 117 117 117 F F F 24 480-499 74 74 74 120 120 120 F F F 25 500-519 76 76 76 124 124 124 F F F 26 520-539 74 74 74 119 119 119 F F F 27 540-559 76 76 76 124 124 124 F F F 28 560-579 76 76 76 124 124 124 F F F 29 580-599 76 76 76 123 123 123 F F F Max gen: 77, Max flush: 125 Begin loop 39 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.804137555Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:32.847421255Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.84797383Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.879110437Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.879746573Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.879779069Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.879793537Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.879807676Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499} {"msg":"Crucible 5f7dc103-88f2-42e5-93be-dbca09afb6f7 has session id: 5f755b7d-c450-4a9e-816d-0a8e4366168e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.880328545Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Upstairs opts: Upstairs UUID: 5f7dc103-88f2-42e5-93be-dbca09afb6f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.880364981Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Crucible stats registered with UUID: 5f7dc103-88f2-42e5-93be-dbca09afb6f7","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.880381609Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910000293Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910031499Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499} {"msg":"5f7dc103-88f2-42e5-93be-dbca09afb6f7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910103821Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910125908Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"0","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910153925Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"0","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910183761Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"1","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910224697Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"1","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910258982Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"2","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910296078Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"2","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910330414Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910593103Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910644497Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"ds_connection connected from Ok(127.0.0.1:53896)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910684493Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40853)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910723888Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37844)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.910758754Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.91113989Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"1","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.911169706Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"2","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.911337157Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"0","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.911867275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"1","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.911897761Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"2","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.911917239Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"0","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.91242407Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912537657Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"[0]R flush_numbers[0..12]: [119, 121, 121, 119, 118, 118, 121, 124, 124, 119, 118, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912666002Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"[0]R generation[0..12]: [74, 75, 75, 74, 72, 72, 75, 76, 76, 74, 72, 77]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912686049Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912700478Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"[1]R flush_numbers[0..12]: [119, 121, 121, 119, 118, 118, 121, 124, 124, 119, 118, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912714426Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"[1]R generation[0..12]: [74, 75, 75, 74, 72, 72, 75, 76, 76, 74, 72, 77]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912728214Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912743383Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"[2]R flush_numbers[0..12]: [119, 121, 121, 119, 118, 118, 121, 124, 124, 119, 118, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912757561Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"[2]R generation[0..12]: [74, 75, 75, 74, 72, 72, 75, 76, 76, 74, 72, 77]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912771249Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912785218Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912798136Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912811265Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Next flush: 126","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:54:32.912828853Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.91284814Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"":"downstairs","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912862739Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912874967Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"0","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912888476Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"1","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912901574Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"client":"2","":"downstairs","session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912914803Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912927101Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"5f7dc103-88f2-42e5-93be-dbca09afb6f7 is now active with session: 5f755b7d-c450-4a9e-816d-0a8e4366168e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.91293996Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912952058Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499,"session_id":"5f755b7d-c450-4a9e-816d-0a8e4366168e"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:32.912964407Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 369 len 24576 data: 4 4 5 5 6 5 02/30 Write block 411 len 20480 data: 8 9 10 8 7 03/30 Read block 478 len 28672 04/30 Write block 538 len 32768 data: 10 10 9 8 6 6 6 7 05/30 Write block 309 len 28672 data: 4 3 3 3 4 4 4 06/30 Write block 346 len 4096 data: 7 07/30 Write block 33 len 32768 data: 4 3 3 4 4 6 7 6 08/30 Write block 399 len 32768 data: 8 7 6 7 7 6 7 7 09/30 Read block 203 len 4096 10/30 Read block 48 len 28672 11/30 Flush 12/30 Write block 531 len 4096 data: 11 13/30 Read block 545 len 12288 14/30 Write block 6 len 32768 data: 6 9 8 6 5 5 6 6 15/30 Write block 203 len 20480 data: 7 8 8 8 9 16/30 Read block 161 len 40960 17/30 Read block 500 len 4096 18/30 Read block 47 len 12288 19/30 Write block 37 len 32768 data: 5 7 8 7 9 9 13 12 20/30 Read block 125 len 20480 21/30 Write block 192 len 8192 data: 6 7 22/30 Read block 293 len 40960 23/30 Write block 123 len 24576 data: 10 9 9 10 10 8 24/30 Write block 98 len 4096 data: 7 25/30 Read block 351 len 36864 26/30 Write block 459 len 12288 data: 7 7 7 27/30 Write block 459 len 8192 data: 8 8 28/30 Write block 546 len 8192 data: 8 6 29/30 Read block 352 len 20480 30/30 Write block 452 len 8192 data: 5 5 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 1 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:32.93119557Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1499} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 119 119 119 T T T 1 020-039 78 78 78 126 126 126 T T T 2 040-059 78 78 78 126 126 126 T T T 3 060-079 74 74 74 119 119 119 F F F 4 080-099 72 72 72 118 118 118 T T T 5 100-119 72 72 72 118 118 118 F F F 6 120-139 75 75 75 121 121 121 T T T 7 140-159 76 76 76 124 124 124 F F F 8 160-179 76 76 76 124 124 124 F F F 9 180-199 74 74 74 119 119 119 T T T 10 200-219 72 72 72 118 118 118 T T T 11 220-239 77 77 77 125 125 125 F F F 12 240-259 72 72 72 118 118 118 F F F 13 260-279 72 72 72 118 118 118 F F F 14 280-299 72 72 72 118 118 118 F F F 15 300-319 78 78 78 126 126 126 F F F 16 320-339 75 75 75 121 121 121 F F F 17 340-359 78 78 78 126 126 126 F F F 18 360-379 78 78 78 126 126 126 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 78 78 78 126 126 126 F F F 21 420-439 75 75 75 121 121 121 F F F 22 440-459 76 76 76 122 122 122 T T T 23 460-479 71 71 71 117 117 117 T T T 24 480-499 74 74 74 120 120 120 F F F 25 500-519 76 76 76 124 124 124 F F F 26 520-539 78 78 78 126 126 126 T T T 27 540-559 78 78 78 126 126 126 T T T 28 560-579 76 76 76 124 124 124 F F F 29 580-599 76 76 76 123 123 123 F F F Max gen: 78, Max flush: 126 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 127 127 127 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 74 74 74 119 119 119 F F F 4 080-099 79 79 79 127 127 127 F F F 5 100-119 72 72 72 118 118 118 F F F 6 120-139 79 79 79 127 127 127 F F F 7 140-159 76 76 76 124 124 124 F F F 8 160-179 76 76 76 124 124 124 F F F 9 180-199 79 79 79 127 127 127 F F F 10 200-219 79 79 79 127 127 127 F F F 11 220-239 77 77 77 125 125 125 F F F 12 240-259 72 72 72 118 118 118 F F F 13 260-279 72 72 72 118 118 118 F F F 14 280-299 72 72 72 118 118 118 F F F 15 300-319 78 78 78 126 126 126 F F F 16 320-339 75 75 75 121 121 121 F F F 17 340-359 78 78 78 126 126 126 F F F 18 360-379 78 78 78 126 126 126 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 78 78 78 126 126 126 F F F 21 420-439 75 75 75 121 121 121 F F F 22 440-459 79 79 79 127 127 127 F F F 23 460-479 79 79 79 127 127 127 F F F 24 480-499 74 74 74 120 120 120 F F F 25 500-519 76 76 76 124 124 124 F F F 26 520-539 79 79 79 127 127 127 F F F 27 540-559 79 79 79 127 127 127 F F F 28 560-579 76 76 76 124 124 124 F F F 29 580-599 76 76 76 123 123 123 F F F Max gen: 79, Max flush: 127 Begin loop 40 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.27824087Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.323805573Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.324767431Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.324807766Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.324822264Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.324837563Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507} {"msg":"Crucible d2718ab1-469b-44a2-8061-1f1209be9a70 has session id: 8457c53f-796d-4a0a-a29a-4ed1e43399ae","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.325267203Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Upstairs opts: Upstairs UUID: d2718ab1-469b-44a2-8061-1f1209be9a70, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.325312067Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Crucible stats registered with UUID: d2718ab1-469b-44a2-8061-1f1209be9a70","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.325333575Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.355727918Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.355783172Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507} {"msg":"d2718ab1-469b-44a2-8061-1f1209be9a70 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.355802229Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.355818218Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"0","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.355846994Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"0","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.355864222Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"1","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.35588339Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"1","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.355899898Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"2","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.355917446Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"2","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.355942603Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.355974939Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.356004936Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:33434)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.356202563Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:52942)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.356254557Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:58962)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.356293952Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.357077381Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"0","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.357125085Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"1","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.357161711Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"2","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.357759671Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"0","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.357804606Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"1","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358047627Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"2","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358385658Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358424083Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"[0]R flush_numbers[0..12]: [127, 127, 127, 119, 127, 118, 127, 124, 124, 127, 127, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358579316Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"[0]R generation[0..12]: [79, 79, 79, 74, 79, 72, 79, 76, 76, 79, 79, 77]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358608722Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.35862479Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"[1]R flush_numbers[0..12]: [127, 127, 127, 119, 127, 118, 127, 124, 124, 127, 127, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358639418Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"[1]R generation[0..12]: [79, 79, 79, 74, 79, 72, 79, 76, 76, 79, 79, 77]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358653507Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358671795Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"[2]R flush_numbers[0..12]: [127, 127, 127, 119, 127, 118, 127, 124, 124, 127, 127, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358685943Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"[2]R generation[0..12]: [79, 79, 79, 74, 79, 72, 79, 76, 76, 79, 79, 77]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358699821Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.35871389Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358726998Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358740257Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:54:37.358753245Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358772463Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"":"downstairs","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358786151Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.35879841Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"0","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358812088Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"1","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358825247Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"client":"2","":"downstairs","session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358838655Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358851014Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"d2718ab1-469b-44a2-8061-1f1209be9a70 is now active with session: 8457c53f-796d-4a0a-a29a-4ed1e43399ae","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358863832Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.35887803Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507,"session_id":"8457c53f-796d-4a0a-a29a-4ed1e43399ae"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:37.358890749Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 557 len 28672 data: 6 6 7 6 7 7 5 02/30 Read block 37 len 36864 03/30 Read block 4 len 8192 04/30 Write block 190 len 8192 data: 8 9 05/30 Read block 107 len 28672 06/30 Write block 410 len 12288 data: 10 9 10 07/30 Read block 278 len 32768 08/30 Write block 276 len 12288 data: 8 7 6 09/30 Write block 572 len 16384 data: 10 8 7 8 10/30 Write block 494 len 36864 data: 4 4 4 4 5 9 12 12 11 11/30 Read block 560 len 28672 12/30 Read block 594 len 20480 13/30 Read block 190 len 36864 14/30 Flush 15/30 Write block 231 len 20480 data: 9 9 6 5 6 16/30 Flush 17/30 Write block 199 len 32768 data: 7 7 7 6 8 9 9 9 18/30 Write block 270 len 8192 data: 7 6 19/30 Write block 459 len 36864 data: 9 9 8 8 9 6 5 6 7 20/30 Write block 517 len 32768 data: 7 7 8 6 4 4 4 4 21/30 Read block 80 len 36864 22/30 Read block 537 len 4096 23/30 Write block 516 len 36864 data: 10 8 8 9 7 5 5 5 5 24/30 Write block 194 len 28672 data: 8 8 7 7 7 8 8 25/30 Read block 568 len 16384 26/30 Read block 476 len 16384 27/30 Write block 100 len 36864 data: 4 4 5 6 7 7 4 5 4 28/30 Read block 241 len 36864 29/30 Read block 167 len 24576 30/30 Write block 302 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 8 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 6 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 13 13 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:37.379380438Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1507} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 127 127 127 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 74 74 74 119 119 119 F F F 4 080-099 79 79 79 127 127 127 F F F 5 100-119 72 72 72 118 118 118 T T T 6 120-139 79 79 79 127 127 127 F F F 7 140-159 76 76 76 124 124 124 F F F 8 160-179 76 76 76 124 124 124 F F F 9 180-199 80 80 80 128 128 128 T T T 10 200-219 79 79 79 127 127 127 T T T 11 220-239 80 80 80 129 129 129 F F F 12 240-259 72 72 72 118 118 118 F F F 13 260-279 80 80 80 128 128 128 T T T 14 280-299 72 72 72 118 118 118 F F F 15 300-319 78 78 78 126 126 126 T T T 16 320-339 75 75 75 121 121 121 F F F 17 340-359 78 78 78 126 126 126 F F F 18 360-379 78 78 78 126 126 126 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 80 80 80 128 128 128 F F F 21 420-439 75 75 75 121 121 121 F F F 22 440-459 79 79 79 127 127 127 T T T 23 460-479 79 79 79 127 127 127 T T T 24 480-499 80 80 80 128 128 128 F F F 25 500-519 80 80 80 128 128 128 T T T 26 520-539 79 79 79 127 127 127 T T T 27 540-559 80 80 80 128 128 128 F F F 28 560-579 80 80 80 128 128 128 F F F 29 580-599 76 76 76 123 123 123 F F F Max gen: 80, Max flush: 129 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 127 127 127 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 74 74 74 119 119 119 F F F 4 080-099 79 79 79 127 127 127 F F F 5 100-119 81 81 81 130 130 130 F F F 6 120-139 79 79 79 127 127 127 F F F 7 140-159 76 76 76 124 124 124 F F F 8 160-179 76 76 76 124 124 124 F F F 9 180-199 81 81 81 130 130 130 F F F 10 200-219 81 81 81 130 130 130 F F F 11 220-239 80 80 80 129 129 129 F F F 12 240-259 72 72 72 118 118 118 F F F 13 260-279 81 81 81 130 130 130 F F F 14 280-299 72 72 72 118 118 118 F F F 15 300-319 81 81 81 130 130 130 F F F 16 320-339 75 75 75 121 121 121 F F F 17 340-359 78 78 78 126 126 126 F F F 18 360-379 78 78 78 126 126 126 F F F 19 380-399 78 78 78 126 126 126 F F F 20 400-419 80 80 80 128 128 128 F F F 21 420-439 75 75 75 121 121 121 F F F 22 440-459 81 81 81 130 130 130 F F F 23 460-479 81 81 81 130 130 130 F F F 24 480-499 80 80 80 128 128 128 F F F 25 500-519 81 81 81 130 130 130 F F F 26 520-539 81 81 81 130 130 130 F F F 27 540-559 80 80 80 128 128 128 F F F 28 560-579 80 80 80 128 128 128 F F F 29 580-599 76 76 76 123 123 123 F F F Max gen: 81, Max flush: 130 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.675794756Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.720633804Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.721415812Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.721469386Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.721487744Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.721505582Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515} {"msg":"Crucible ac5dc8f2-08cf-4777-b8c9-9dd1c0df0758 has session id: 36b6611f-1717-4258-9f77-6f6bf083f7a9","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.721887707Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Upstairs opts: Upstairs UUID: ac5dc8f2-08cf-4777-b8c9-9dd1c0df0758, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.722047159Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Crucible stats registered with UUID: ac5dc8f2-08cf-4777-b8c9-9dd1c0df0758","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.722086654Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.75262726Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.752672805Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515} {"msg":"ac5dc8f2-08cf-4777-b8c9-9dd1c0df0758 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.752741907Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.752786832Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"0","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.75288676Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"0","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.752931175Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"1","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.75297049Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"1","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.753006956Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"2","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.753049751Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"2","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.753172117Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.753206173Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.75323025Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:40879)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.753263096Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:39178)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.753284314Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:47012)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.753302152Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.75374758Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"0","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.753786485Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"1","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.753894203Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"2","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.754436739Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"0","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.754487143Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"1","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.754574793Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"2","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755049318Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755083374Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"[0]R flush_numbers[0..12]: [127, 127, 127, 119, 127, 130, 127, 124, 124, 130, 130, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755193131Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"[0]R generation[0..12]: [79, 79, 79, 74, 79, 81, 79, 76, 76, 81, 81, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755214838Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755229757Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"[1]R flush_numbers[0..12]: [127, 127, 127, 119, 127, 130, 127, 124, 124, 130, 130, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755244025Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"[1]R generation[0..12]: [79, 79, 79, 74, 79, 81, 79, 76, 76, 81, 81, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755257893Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755272122Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"[2]R flush_numbers[0..12]: [127, 127, 127, 119, 127, 130, 127, 124, 124, 130, 130, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.75528591Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"[2]R generation[0..12]: [79, 79, 79, 74, 79, 81, 79, 76, 76, 81, 81, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755299599Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755313517Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755326735Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755339874Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755352912Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"All extents match","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO ","level":30,"time":"2025-01-13T15:54:41.75537272Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"":"downstairs","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755390138Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755402847Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"0","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755416285Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"1","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755429283Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"client":"2","":"downstairs","session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755442432Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755454701Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"ac5dc8f2-08cf-4777-b8c9-9dd1c0df0758 is now active with session: 36b6611f-1717-4258-9f77-6f6bf083f7a9","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755467679Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755479747Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515,"session_id":"36b6611f-1717-4258-9f77-6f6bf083f7a9"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:41.755492026Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 599 len 4096 data: 4 02/30 Read block 338 len 16384 03/30 Flush 04/30 Write block 126 len 12288 data: 11 11 9 05/30 Read block 98 len 28672 06/30 Read block 260 len 20480 07/30 Write block 412 len 40960 data: 11 11 9 8 8 6 5 6 6 6 08/30 Write block 108 len 20480 data: 5 3 3 3 3 09/30 Read block 176 len 20480 10/30 Read block 60 len 36864 11/30 Read block 130 len 36864 12/30 Write block 80 len 40960 data: 3 3 8 8 7 9 10 11 11 15 13/30 Read block 507 len 32768 14/30 Write block 170 len 12288 data: 12 9 7 15/30 Write block 515 len 20480 data: 7 11 9 9 10 16/30 Read block 514 len 32768 17/30 Write block 350 len 32768 data: 7 5 4 3 6 6 6 6 18/30 Read block 335 len 4096 19/30 Read block 384 len 32768 20/30 Write block 105 len 24576 data: 8 5 6 6 4 4 21/30 Write block 397 len 20480 data: 8 7 9 8 7 22/30 Read block 96 len 16384 23/30 Write block 342 len 32768 data: 8 8 8 8 8 6 7 6 24/30 Flush 25/30 Write block 198 len 8192 data: 8 9 26/30 Read block 297 len 12288 27/30 Read block 501 len 8192 28/30 Read block 141 len 28672 29/30 Write block 254 len 24576 data: 9 9 6 7 7 7 30/30 Read block 68 len 20480 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 4 6 6 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:41.776069235Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1515} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 127 127 127 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 74 74 74 119 119 119 F F F 4 080-099 82 82 82 132 132 132 F F F 5 100-119 82 82 82 132 132 132 F F F 6 120-139 82 82 82 132 132 132 F F F 7 140-159 76 76 76 124 124 124 F F F 8 160-179 82 82 82 132 132 132 F F F 9 180-199 81 81 81 130 130 130 T T T 10 200-219 81 81 81 130 130 130 F F F 11 220-239 80 80 80 129 129 129 F F F 12 240-259 72 72 72 118 118 118 T T T 13 260-279 81 81 81 130 130 130 F F F 14 280-299 72 72 72 118 118 118 F F F 15 300-319 81 81 81 130 130 130 F F F 16 320-339 75 75 75 121 121 121 F F F 17 340-359 82 82 82 132 132 132 F F F 18 360-379 78 78 78 126 126 126 F F F 19 380-399 82 82 82 132 132 132 F F F 20 400-419 82 82 82 132 132 132 F F F 21 420-439 82 82 82 132 132 132 F F F 22 440-459 81 81 81 130 130 130 F F F 23 460-479 81 81 81 130 130 130 F F F 24 480-499 80 80 80 128 128 128 F F F 25 500-519 82 82 82 132 132 132 F F F 26 520-539 81 81 81 130 130 130 F F F 27 540-559 80 80 80 128 128 128 F F F 28 560-579 80 80 80 128 128 128 F F F 29 580-599 82 82 82 131 131 131 F F F Max gen: 82, Max flush: 132 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 127 127 127 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 74 74 74 119 119 119 F F F 4 080-099 82 82 82 132 132 132 F F F 5 100-119 82 82 82 132 132 132 F F F 6 120-139 82 82 82 132 132 132 F F F 7 140-159 76 76 76 124 124 124 F F F 8 160-179 82 82 82 132 132 132 F F F 9 180-199 83 83 83 133 133 133 F F F 10 200-219 81 81 81 130 130 130 F F F 11 220-239 80 80 80 129 129 129 F F F 12 240-259 83 83 83 133 133 133 F F F 13 260-279 81 81 81 130 130 130 F F F 14 280-299 72 72 72 118 118 118 F F F 15 300-319 81 81 81 130 130 130 F F F 16 320-339 75 75 75 121 121 121 F F F 17 340-359 82 82 82 132 132 132 F F F 18 360-379 78 78 78 126 126 126 F F F 19 380-399 82 82 82 132 132 132 F F F 20 400-419 82 82 82 132 132 132 F F F 21 420-439 82 82 82 132 132 132 F F F 22 440-459 81 81 81 130 130 130 F F F 23 460-479 81 81 81 130 130 130 F F F 24 480-499 80 80 80 128 128 128 F F F 25 500-519 82 82 82 132 132 132 F F F 26 520-539 81 81 81 130 130 130 F F F 27 540-559 80 80 80 128 128 128 F F F 28 560-579 80 80 80 128 128 128 F F F 29 580-599 82 82 82 131 131 131 F F F Max gen: 83, Max flush: 133 Begin loop 42 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.04087858Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.08663747Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.087610917Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.087694707Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.087726603Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.08775341Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523} {"msg":"Crucible e5735ec5-9dd5-426c-ba71-ca100c349324 has session id: 12a536ec-456c-49bc-9d87-f308d8565f44","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.08818489Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Upstairs opts: Upstairs UUID: e5735ec5-9dd5-426c-ba71-ca100c349324, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.088233044Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Crucible stats registered with UUID: e5735ec5-9dd5-426c-ba71-ca100c349324","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.088257941Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.1184338Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.118488044Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523} {"msg":"e5735ec5-9dd5-426c-ba71-ca100c349324 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.118504332Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.11852051Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"0","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.118565425Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"0","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.118601301Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"1","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.118637006Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"1","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.118671712Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"2","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.118705738Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"2","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.118739594Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.11877337Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.118809036Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"ds_connection connected from Ok(127.0.0.1:57653)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.118843882Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"ds_connection connected from Ok(127.0.0.1:52069)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.118879428Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"ds_connection connected from Ok(127.0.0.1:65085)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.118915124Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.119654218Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"0","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.119683934Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"2","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.119907938Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"1","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.12040718Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"0","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.120442426Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"2","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.120459874Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"1","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.12100437Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121033017Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"[0]R flush_numbers[0..12]: [127, 127, 127, 119, 132, 132, 132, 124, 132, 133, 130, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121098549Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"[0]R generation[0..12]: [79, 79, 79, 74, 82, 82, 82, 76, 82, 83, 81, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121120187Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121135475Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"[1]R flush_numbers[0..12]: [127, 127, 127, 119, 132, 132, 132, 124, 132, 133, 130, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121149673Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"[1]R generation[0..12]: [79, 79, 79, 74, 82, 82, 82, 76, 82, 83, 81, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121163431Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.12117757Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"[2]R flush_numbers[0..12]: [127, 127, 127, 119, 132, 132, 132, 124, 132, 133, 130, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121191668Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"[2]R generation[0..12]: [79, 79, 79, 74, 82, 82, 82, 76, 82, 83, 81, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121205786Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121219655Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121236913Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121251051Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.1212643Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121283797Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"":"downstairs","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121297136Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121309245Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"0","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121322623Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"1","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121336331Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"client":"2","":"downstairs","session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.1213495Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121361788Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"e5735ec5-9dd5-426c-ba71-ca100c349324 is now active with session: 12a536ec-456c-49bc-9d87-f308d8565f44","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121374547Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121386805Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523,"session_id":"12a536ec-456c-49bc-9d87-f308d8565f44"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:46.121399184Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 461 len 20480 data: 9 9 10 7 6 02/30 Write block 213 len 16384 data: 10 9 9 9 03/30 Write block 588 len 32768 data: 8 10 10 12 11 10 9 7 04/30 Flush 05/30 Write block 522 len 4096 data: 6 06/30 Read block 506 len 40960 07/30 Write block 403 len 32768 data: 8 7 8 8 6 8 8 11 08/30 Write block 67 len 12288 data: 9 9 8 09/30 Read block 84 len 20480 10/30 Flush 11/30 Write block 397 len 8192 data: 9 8 12/30 Write block 310 len 4096 data: 4 13/30 Read block 474 len 36864 14/30 Read block 401 len 36864 15/30 Write block 437 len 12288 data: 8 8 6 16/30 Read block 313 len 32768 17/30 Read block 189 len 8192 18/30 Write block 517 len 20480 data: 10 10 11 8 6 19/30 Write block 15 len 12288 data: 4 7 7 20/30 Read block 111 len 28672 21/30 Write block 455 len 40960 data: 6 7 7 7 10 10 10 10 11 8 22/30 Write block 359 len 12288 data: 9 7 6 23/30 Flush 24/30 Write block 476 len 20480 data: 6 5 8 8 7 25/30 Read block 235 len 28672 26/30 Write block 161 len 32768 data: 12 13 11 11 9 11 12 13 27/30 Write block 211 len 32768 data: 9 10 11 10 10 10 8 10 28/30 Write block 167 len 24576 data: 13 14 12 13 10 8 29/30 Read block 505 len 24576 30/30 Write block 180 len 16384 data: 8 7 5 6 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:46.140186702Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1523} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 136 136 136 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 84 84 84 135 135 135 F F F 4 080-099 82 82 82 132 132 132 F F F 5 100-119 82 82 82 132 132 132 F F F 6 120-139 82 82 82 132 132 132 F F F 7 140-159 76 76 76 124 124 124 F F F 8 160-179 82 82 82 132 132 132 T T T 9 180-199 83 83 83 133 133 133 T T T 10 200-219 84 84 84 134 134 134 T T T 11 220-239 80 80 80 129 129 129 F F F 12 240-259 83 83 83 133 133 133 F F F 13 260-279 81 81 81 130 130 130 F F F 14 280-299 72 72 72 118 118 118 F F F 15 300-319 84 84 84 136 136 136 F F F 16 320-339 75 75 75 121 121 121 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 84 84 84 136 136 136 F F F 19 380-399 84 84 84 136 136 136 F F F 20 400-419 84 84 84 135 135 135 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 84 84 84 136 136 136 F F F 23 460-479 84 84 84 136 136 136 T T T 24 480-499 80 80 80 128 128 128 T T T 25 500-519 84 84 84 136 136 136 F F F 26 520-539 84 84 84 136 136 136 F F F 27 540-559 80 80 80 128 128 128 F F F 28 560-579 80 80 80 128 128 128 F F F 29 580-599 84 84 84 134 134 134 F F F Max gen: 84, Max flush: 136 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 136 136 136 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 84 84 84 135 135 135 F F F 4 080-099 82 82 82 132 132 132 F F F 5 100-119 82 82 82 132 132 132 F F F 6 120-139 82 82 82 132 132 132 F F F 7 140-159 76 76 76 124 124 124 F F F 8 160-179 85 85 85 137 137 137 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 85 85 85 137 137 137 F F F 11 220-239 80 80 80 129 129 129 F F F 12 240-259 83 83 83 133 133 133 F F F 13 260-279 81 81 81 130 130 130 F F F 14 280-299 72 72 72 118 118 118 F F F 15 300-319 84 84 84 136 136 136 F F F 16 320-339 75 75 75 121 121 121 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 84 84 84 136 136 136 F F F 19 380-399 84 84 84 136 136 136 F F F 20 400-419 84 84 84 135 135 135 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 84 84 84 136 136 136 F F F 23 460-479 85 85 85 137 137 137 F F F 24 480-499 85 85 85 137 137 137 F F F 25 500-519 84 84 84 136 136 136 F F F 26 520-539 84 84 84 136 136 136 F F F 27 540-559 80 80 80 128 128 128 F F F 28 560-579 80 80 80 128 128 128 F F F 29 580-599 84 84 84 134 134 134 F F F Max gen: 85, Max flush: 137 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.427631936Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.472851309Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.473921854Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.47396182Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.473977068Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.473991766Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531} {"msg":"Crucible 62dfd6d4-5e1a-4255-8d49-4cabc901a8c4 has session id: d7c72e9a-fd80-4d6b-8541-ab7f37714cee","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.474539302Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Upstairs opts: Upstairs UUID: 62dfd6d4-5e1a-4255-8d49-4cabc901a8c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.474585577Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Crucible stats registered with UUID: 62dfd6d4-5e1a-4255-8d49-4cabc901a8c4","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.474607044Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.505560173Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.505620885Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531} {"msg":"62dfd6d4-5e1a-4255-8d49-4cabc901a8c4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.505636664Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.505652902Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"0","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.505683258Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"0","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.505700796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"1","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.505719834Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"1","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.505736802Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"2","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.50575486Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"2","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.505771828Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.505788906Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.505805934Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"ds_connection connected from Ok(127.0.0.1:33069)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.506020839Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"ds_connection connected from Ok(127.0.0.1:43846)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.506060644Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"ds_connection connected from Ok(127.0.0.1:56990)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.50609441Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.506961089Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"2","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.506996485Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"0","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.507156366Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"1","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.507648729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"0","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.507678905Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"2","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.507787883Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"1","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508244629Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508359206Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"[0]R flush_numbers[0..12]: [136, 127, 127, 135, 132, 132, 132, 124, 137, 137, 137, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508451385Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"[0]R generation[0..12]: [84, 79, 79, 84, 82, 82, 82, 76, 85, 85, 85, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508471923Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508486501Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"[1]R flush_numbers[0..12]: [136, 127, 127, 135, 132, 132, 132, 124, 137, 137, 137, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.5085005Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"[1]R generation[0..12]: [84, 79, 79, 84, 82, 82, 82, 76, 85, 85, 85, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508514178Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508532216Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"[2]R flush_numbers[0..12]: [136, 127, 127, 135, 132, 132, 132, 124, 137, 137, 137, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508546604Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"[2]R generation[0..12]: [84, 79, 79, 84, 82, 82, 82, 76, 85, 85, 85, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508560442Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508574381Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508587539Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508600868Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508613896Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:54:50.508626925Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"":"downstairs","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508644103Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508656151Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"0","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.50866941Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"1","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508683558Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"client":"2","":"downstairs","session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508696857Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508709095Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"62dfd6d4-5e1a-4255-8d49-4cabc901a8c4 is now active with session: d7c72e9a-fd80-4d6b-8541-ab7f37714cee","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508721844Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508733872Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531,"session_id":"d7c72e9a-fd80-4d6b-8541-ab7f37714cee"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:50.508748041Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 383 len 36864 data: 8 8 7 7 4 5 8 8 8 02/30 Write block 258 len 16384 data: 8 8 9 9 03/30 Read block 435 len 32768 04/30 Flush 05/30 Write block 294 len 40960 data: 8 6 4 6 6 6 5 5 8 7 06/30 Write block 31 len 4096 data: 5 07/30 Read block 492 len 12288 08/30 Write block 401 len 12288 data: 8 8 9 09/30 Write block 381 len 16384 data: 7 8 9 9 10/30 Write block 237 len 24576 data: 6 7 5 4 6 6 11/30 Write block 117 len 20480 data: 7 6 7 6 6 12/30 Read block 406 len 8192 13/30 Read block 469 len 8192 14/30 Read block 295 len 20480 15/30 Read block 57 len 16384 16/30 Read block 8 len 12288 17/30 Read block 207 len 32768 18/30 Read block 240 len 28672 19/30 Write block 10 len 28672 data: 6 6 7 7 6 5 8 20/30 Write block 16 len 16384 data: 9 8 9 9 21/30 Write block 486 len 20480 data: 7 7 7 7 6 22/30 Write block 127 len 12288 data: 12 10 6 23/30 Read block 330 len 12288 24/30 Read block 443 len 20480 25/30 Read block 347 len 8192 26/30 Read block 340 len 24576 27/30 Write block 87 len 32768 data: 12 12 16 14 15 14 14 11 28/30 Write block 73 len 12288 data: 4 4 3 29/30 Write block 577 len 8192 data: 7 7 30/30 Read block 57 len 8192 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 10 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 8 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 4 6 Done 26 24 22 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:50.527296706Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1531} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 136 136 136 T T T 1 020-039 79 79 79 127 127 127 T T T 2 040-059 79 79 79 127 127 127 F F F 3 060-079 84 84 84 135 135 135 T T T 4 080-099 82 82 82 132 132 132 T T T 5 100-119 82 82 82 132 132 132 T T T 6 120-139 82 82 82 132 132 132 T T T 7 140-159 76 76 76 124 124 124 F F F 8 160-179 85 85 85 137 137 137 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 85 85 85 137 137 137 F F F 11 220-239 80 80 80 129 129 129 T T T 12 240-259 86 86 86 138 138 138 T T T 13 260-279 86 86 86 138 138 138 F F F 14 280-299 72 72 72 118 118 118 T T T 15 300-319 84 84 84 136 136 136 T T T 16 320-339 75 75 75 121 121 121 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 84 84 84 136 136 136 F F F 19 380-399 86 86 86 138 138 138 T T T 20 400-419 84 84 84 135 135 135 T T T 21 420-439 84 84 84 136 136 136 F F F 22 440-459 84 84 84 136 136 136 F F F 23 460-479 85 85 85 137 137 137 F F F 24 480-499 85 85 85 137 137 137 T T T 25 500-519 84 84 84 136 136 136 F F F 26 520-539 84 84 84 136 136 136 F F F 27 540-559 80 80 80 128 128 128 F F F 28 560-579 80 80 80 128 128 128 T T T 29 580-599 84 84 84 134 134 134 F F F Max gen: 86, Max flush: 138 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 139 139 139 F F F 1 020-039 87 87 87 139 139 139 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 87 87 87 139 139 139 F F F 4 080-099 87 87 87 139 139 139 F F F 5 100-119 87 87 87 139 139 139 F F F 6 120-139 87 87 87 139 139 139 F F F 7 140-159 76 76 76 124 124 124 F F F 8 160-179 85 85 85 137 137 137 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 85 85 85 137 137 137 F F F 11 220-239 87 87 87 139 139 139 F F F 12 240-259 87 87 87 139 139 139 F F F 13 260-279 86 86 86 138 138 138 F F F 14 280-299 87 87 87 139 139 139 F F F 15 300-319 87 87 87 139 139 139 F F F 16 320-339 75 75 75 121 121 121 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 84 84 84 136 136 136 F F F 19 380-399 87 87 87 139 139 139 F F F 20 400-419 87 87 87 139 139 139 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 84 84 84 136 136 136 F F F 23 460-479 85 85 85 137 137 137 F F F 24 480-499 87 87 87 139 139 139 F F F 25 500-519 84 84 84 136 136 136 F F F 26 520-539 84 84 84 136 136 136 F F F 27 540-559 80 80 80 128 128 128 F F F 28 560-579 87 87 87 139 139 139 F F F 29 580-599 84 84 84 134 134 134 F F F Max gen: 87, Max flush: 139 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.853139609Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.897042246Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.897736445Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.897773221Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.897787669Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.897801918Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539} {"msg":"Crucible bd27cb13-8838-4c24-b813-4fa5f476c5e5 has session id: 95f00d09-df3c-4888-bbf0-966209230927","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.898257834Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Upstairs opts: Upstairs UUID: bd27cb13-8838-4c24-b813-4fa5f476c5e5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.898302639Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Crucible stats registered with UUID: bd27cb13-8838-4c24-b813-4fa5f476c5e5","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.898326756Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.928619382Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.928674245Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539} {"msg":"bd27cb13-8838-4c24-b813-4fa5f476c5e5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.928693043Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.928709871Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"0","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.928739058Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"0","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.928756756Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"1","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.928774663Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"1","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.928796621Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"2","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.928814989Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"2","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.928845985Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.928880311Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.928913957Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"ds_connection connected from Ok(127.0.0.1:63335)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.929075878Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"ds_connection connected from Ok(127.0.0.1:65229)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.929179106Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"ds_connection connected from Ok(127.0.0.1:61801)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.929328449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.929948706Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"1","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.929991381Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"0","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.930024438Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"2","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.930578833Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"0","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.930612269Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"1","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.930640796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"2","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931187752Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931225148Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"[0]R flush_numbers[0..12]: [139, 139, 127, 139, 139, 139, 139, 124, 137, 137, 137, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931272692Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"[0]R generation[0..12]: [87, 87, 79, 87, 87, 87, 87, 76, 85, 85, 85, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.93129039Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931306358Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"[1]R flush_numbers[0..12]: [139, 139, 127, 139, 139, 139, 139, 124, 137, 137, 137, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931320386Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"[1]R generation[0..12]: [87, 87, 79, 87, 87, 87, 87, 76, 85, 85, 85, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931334105Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931348173Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"[2]R flush_numbers[0..12]: [139, 139, 127, 139, 139, 139, 139, 124, 137, 137, 137, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931361912Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"[2]R generation[0..12]: [87, 87, 79, 87, 87, 87, 87, 76, 85, 85, 85, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.93137538Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931389378Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931403227Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931416565Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931429434Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931446032Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"":"downstairs","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.93145962Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-01-13T15:54:54.931479208Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"0","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931501405Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"1","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931525853Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"client":"2","":"downstairs","session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931540601Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931558069Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"bd27cb13-8838-4c24-b813-4fa5f476c5e5 is now active with session: 95f00d09-df3c-4888-bbf0-966209230927","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931582836Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:54.931608233Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539,"session_id":"95f00d09-df3c-4888-bbf0-966209230927"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-13T15:54:54.93163381Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 19 len 36864 data: 10 7 7 7 7 7 7 7 6 02/30 Write block 384 len 16384 data: 10 8 8 5 03/30 Read block 357 len 32768 04/30 Write block 494 len 12288 data: 5 5 5 05/30 Write block 149 len 28672 data: 7 6 7 6 6 6 6 06/30 Write block 551 len 28672 data: 10 8 8 5 7 6 7 07/30 Flush 08/30 Write block 211 len 8192 data: 10 11 09/30 Read block 462 len 16384 10/30 Read block 572 len 8192 11/30 Flush 12/30 Write block 379 len 16384 data: 5 6 8 9 13/30 Read block 289 len 4096 14/30 Read block 576 len 8192 15/30 Write block 8 len 20480 data: 9 7 7 7 8 16/30 Read block 377 len 28672 17/30 Write block 85 len 20480 data: 10 11 13 13 17 18/30 Flush 19/30 Write block 23 len 36864 data: 8 8 8 8 7 6 6 5 6 20/30 Read block 332 len 20480 21/30 Read block 250 len 28672 22/30 Read block 193 len 24576 23/30 Write block 429 len 20480 data: 4 3 4 6 5 24/30 Read block 419 len 20480 25/30 Write block 148 len 40960 data: 7 8 7 8 7 7 7 7 6 7 26/30 Read block 131 len 4096 27/30 Write block 18 len 36864 data: 10 11 8 8 8 9 9 9 9 28/30 Write block 509 len 20480 data: 6 6 6 6 5 29/30 Write block 310 len 20480 data: 5 4 4 5 5 30/30 Read block 71 len 4096 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 5 Sent Sent Done false 1029 Acked Read 1 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 10 10 12 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:54.951351429Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1539} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 142 142 142 T T T 1 020-039 88 88 88 140 140 140 T T T 2 040-059 79 79 79 127 127 127 F F F 3 060-079 87 87 87 139 139 139 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 87 87 87 139 139 139 F F F 6 120-139 87 87 87 139 139 139 F F F 7 140-159 88 88 88 140 140 140 T T T 8 160-179 85 85 85 137 137 137 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 88 88 88 141 141 141 F F F 11 220-239 87 87 87 139 139 139 F F F 12 240-259 87 87 87 139 139 139 F F F 13 260-279 86 86 86 138 138 138 F F F 14 280-299 87 87 87 139 139 139 F F F 15 300-319 87 87 87 139 139 139 T T T 16 320-339 75 75 75 121 121 121 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 88 88 88 142 142 142 F F F 19 380-399 88 88 88 142 142 142 F F F 20 400-419 87 87 87 139 139 139 F F F 21 420-439 84 84 84 136 136 136 T T T 22 440-459 84 84 84 136 136 136 F F F 23 460-479 85 85 85 137 137 137 F F F 24 480-499 88 88 88 140 140 140 F F F 25 500-519 84 84 84 136 136 136 T T T 26 520-539 84 84 84 136 136 136 F F F 27 540-559 88 88 88 140 140 140 F F F 28 560-579 87 87 87 139 139 139 F F F 29 580-599 84 84 84 134 134 134 F F F Max gen: 88, Max flush: 142 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 143 143 143 F F F 1 020-039 89 89 89 143 143 143 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 87 87 87 139 139 139 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 87 87 87 139 139 139 F F F 6 120-139 87 87 87 139 139 139 F F F 7 140-159 89 89 89 143 143 143 F F F 8 160-179 85 85 85 137 137 137 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 88 88 88 141 141 141 F F F 11 220-239 87 87 87 139 139 139 F F F 12 240-259 87 87 87 139 139 139 F F F 13 260-279 86 86 86 138 138 138 F F F 14 280-299 87 87 87 139 139 139 F F F 15 300-319 89 89 89 143 143 143 F F F 16 320-339 75 75 75 121 121 121 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 88 88 88 142 142 142 F F F 19 380-399 88 88 88 142 142 142 F F F 20 400-419 87 87 87 139 139 139 F F F 21 420-439 89 89 89 143 143 143 F F F 22 440-459 84 84 84 136 136 136 F F F 23 460-479 85 85 85 137 137 137 F F F 24 480-499 88 88 88 140 140 140 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 84 84 84 136 136 136 F F F 27 540-559 88 88 88 140 140 140 F F F 28 560-579 87 87 87 139 139 139 F F F 29 580-599 84 84 84 134 134 134 F F F Max gen: 89, Max flush: 143 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.235062759Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:59.279142345Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.279542209Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.311228531Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.311870227Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.311907062Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.31192284Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.311937149Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547} {"msg":"Crucible 9b49668f-f953-4d0c-bb2e-024d49c97b9e has session id: 141a1edb-44e3-4763-985c-aaffec6ca575","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.312448599Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Upstairs opts: Upstairs UUID: 9b49668f-f953-4d0c-bb2e-024d49c97b9e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.312493324Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Crucible stats registered with UUID: 9b49668f-f953-4d0c-bb2e-024d49c97b9e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.312521391Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.342624058Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.34268756Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547} {"msg":"9b49668f-f953-4d0c-bb2e-024d49c97b9e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.342710858Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.342728106Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"0","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.34277531Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"0","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.342810836Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"1","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.342846532Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"1","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.342879048Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"2","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.342914464Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"2","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.3429536Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.342982986Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.343012333Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"ds_connection connected from Ok(127.0.0.1:55955)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.34303786Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"ds_connection connected from Ok(127.0.0.1:46838)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.343063727Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"ds_connection connected from Ok(127.0.0.1:57661)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.343260214Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.344031874Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"1","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.344228661Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"0","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.344272396Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"2","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.344651022Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"1","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.344693826Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"2","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.344971364Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"0","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345240953Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345268099Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"[0]R flush_numbers[0..12]: [143, 143, 127, 139, 142, 139, 139, 143, 137, 137, 141, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345588562Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"[0]R generation[0..12]: [89, 89, 79, 87, 88, 87, 87, 89, 85, 85, 88, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.34561017Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345625068Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"[1]R flush_numbers[0..12]: [143, 143, 127, 139, 142, 139, 139, 143, 137, 137, 141, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345639246Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"[1]R generation[0..12]: [89, 89, 79, 87, 88, 87, 87, 89, 85, 85, 88, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345653105Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345671552Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"[2]R flush_numbers[0..12]: [143, 143, 127, 139, 142, 139, 139, 143, 137, 137, 141, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.34568683Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"[2]R generation[0..12]: [89, 89, 79, 87, 88, 87, 87, 89, 85, 85, 88, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345700659Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345714987Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345728286Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345741654Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345754743Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345767941Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"":"downstairs","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345787139Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345822135Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"0","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345849842Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"1","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345869979Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"client":"2","":"downstairs","session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345884218Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345898076Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"9b49668f-f953-4d0c-bb2e-024d49c97b9e is now active with session: 141a1edb-44e3-4763-985c-aaffec6ca575","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345922443Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.34595093Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547,"session_id":"141a1edb-44e3-4763-985c-aaffec6ca575"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:54:59.345977317Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 320 len 36864 data: 6 6 7 7 6 8 6 5 6 02/30 Write block 528 len 20480 data: 6 6 8 12 12 03/30 Read block 137 len 28672 04/30 Flush 05/30 Write block 54 len 24576 data: 9 9 6 6 6 6 06/30 Read block 247 len 12288 07/30 Flush 08/30 Write block 13 len 24576 data: 8 7 6 10 9 11 09/30 Read block 265 len 12288 10/30 Write block 547 len 4096 data: 7 11/30 Read block 104 len 8192 12/30 Read block 366 len 12288 13/30 Read block 13 len 20480 14/30 Write block 562 len 20480 data: 8 6 6 6 7 15/30 Read block 540 len 4096 16/30 Write block 437 len 4096 data: 9 17/30 Read block 555 len 12288 18/30 Read block 76 len 4096 19/30 Write block 19 len 24576 data: 12 9 9 9 10 10 20/30 Read block 539 len 20480 21/30 Read block 415 len 32768 22/30 Write block 571 len 16384 data: 7 11 9 8 23/30 Read block 311 len 20480 24/30 Read block 377 len 8192 25/30 Write block 432 len 16384 data: 7 6 5 5 26/30 Write block 245 len 16384 data: 5 5 5 6 27/30 Write block 36 len 28672 data: 5 6 8 9 8 10 10 28/30 Read block 551 len 36864 29/30 Write block 293 len 8192 data: 9 9 30/30 Write block 144 len 40960 data: 11 8 6 6 8 9 8 9 8 8 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 6 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:54:59.365616355Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1547} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 143 143 143 T T T 1 020-039 89 89 89 143 143 143 T T T 2 040-059 90 90 90 145 145 145 T T T 3 060-079 87 87 87 139 139 139 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 87 87 87 139 139 139 F F F 6 120-139 87 87 87 139 139 139 F F F 7 140-159 89 89 89 143 143 143 T T T 8 160-179 85 85 85 137 137 137 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 88 88 88 141 141 141 F F F 11 220-239 87 87 87 139 139 139 F F F 12 240-259 87 87 87 139 139 139 T T T 13 260-279 86 86 86 138 138 138 F F F 14 280-299 87 87 87 139 139 139 T T T 15 300-319 89 89 89 143 143 143 F F F 16 320-339 90 90 90 144 144 144 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 88 88 88 142 142 142 F F F 19 380-399 88 88 88 142 142 142 F F F 20 400-419 87 87 87 139 139 139 F F F 21 420-439 89 89 89 143 143 143 T T T 22 440-459 84 84 84 136 136 136 F F F 23 460-479 85 85 85 137 137 137 F F F 24 480-499 88 88 88 140 140 140 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 90 90 90 144 144 144 F F F 27 540-559 88 88 88 140 140 140 T T T 28 560-579 87 87 87 139 139 139 T T T 29 580-599 84 84 84 134 134 134 F F F Max gen: 90, Max flush: 145 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 146 146 146 F F F 1 020-039 91 91 91 146 146 146 F F F 2 040-059 91 91 91 146 146 146 F F F 3 060-079 87 87 87 139 139 139 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 87 87 87 139 139 139 F F F 6 120-139 87 87 87 139 139 139 F F F 7 140-159 91 91 91 146 146 146 F F F 8 160-179 85 85 85 137 137 137 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 88 88 88 141 141 141 F F F 11 220-239 87 87 87 139 139 139 F F F 12 240-259 91 91 91 146 146 146 F F F 13 260-279 86 86 86 138 138 138 F F F 14 280-299 91 91 91 146 146 146 F F F 15 300-319 89 89 89 143 143 143 F F F 16 320-339 90 90 90 144 144 144 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 88 88 88 142 142 142 F F F 19 380-399 88 88 88 142 142 142 F F F 20 400-419 87 87 87 139 139 139 F F F 21 420-439 91 91 91 146 146 146 F F F 22 440-459 84 84 84 136 136 136 F F F 23 460-479 85 85 85 137 137 137 F F F 24 480-499 88 88 88 140 140 140 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 90 90 90 144 144 144 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 91 91 91 146 146 146 F F F 29 580-599 84 84 84 134 134 134 F F F Max gen: 91, Max flush: 146 Begin loop 46 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.6923746Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.73591456Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.736652134Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.736695009Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.736709417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.736725245Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555} {"msg":"Crucible 28aa47b6-2172-4af7-87ce-f717f0c7b9a9 has session id: ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.737245034Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Upstairs opts: Upstairs UUID: 28aa47b6-2172-4af7-87ce-f717f0c7b9a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.73728146Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Crucible stats registered with UUID: 28aa47b6-2172-4af7-87ce-f717f0c7b9a9","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.737297398Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767111069Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767154124Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555} {"msg":"28aa47b6-2172-4af7-87ce-f717f0c7b9a9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767185161Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767202289Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"0","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767230426Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"0","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767250873Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"1","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767284399Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"1","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767329124Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"2","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767369979Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"2","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767412784Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.76744722Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767482796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"ds_connection connected from Ok(127.0.0.1:36215)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767724318Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"ds_connection connected from Ok(127.0.0.1:56047)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767775492Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"ds_connection connected from Ok(127.0.0.1:48122)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.767808108Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.768343276Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"1","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.76839217Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"0","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.768431355Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"2","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769009938Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"1","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769041054Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"0","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769189077Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"2","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769611258Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769642974Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"[0]R flush_numbers[0..12]: [146, 146, 146, 139, 142, 139, 139, 146, 137, 137, 141, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769789017Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"[0]R generation[0..12]: [91, 91, 91, 87, 88, 87, 87, 91, 85, 85, 88, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769811354Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769826473Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"[1]R flush_numbers[0..12]: [146, 146, 146, 139, 142, 139, 139, 146, 137, 137, 141, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769840771Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"[1]R generation[0..12]: [91, 91, 91, 87, 88, 87, 87, 91, 85, 85, 88, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769854699Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769870187Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"[2]R flush_numbers[0..12]: [146, 146, 146, 139, 142, 139, 139, 146, 137, 137, 141, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769884436Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"[2]R generation[0..12]: [91, 91, 91, 87, 88, 87, 87, 91, 85, 85, 88, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769898214Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769912193Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769925381Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769938619Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769951618Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.769964716Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"":"downstairs","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.770010381Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.770040178Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"0","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.770068084Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"1","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.770095671Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"client":"2","":"downstairs","session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.770124038Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.770150795Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"28aa47b6-2172-4af7-87ce-f717f0c7b9a9 is now active with session: ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.770177692Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:03.770204079Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555,"session_id":"ca5afa1c-0fad-4aa7-bcfb-67ea2afafc01"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-13T15:55:03.770230656Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 409 len 28672 data: 9 12 10 12 12 10 9 02/30 Read block 443 len 40960 03/30 Write block 479 len 16384 data: 9 8 6 4 04/30 Write block 479 len 16384 data: 10 9 7 5 05/30 Flush 06/30 Write block 70 len 24576 data: 8 7 5 5 5 4 07/30 Read block 163 len 24576 08/30 Read block 375 len 32768 09/30 Read block 176 len 16384 10/30 Write block 182 len 32768 data: 6 7 7 6 7 6 7 9 11/30 Write block 464 len 36864 data: 9 7 7 8 7 6 4 4 4 12/30 Write block 455 len 32768 data: 7 8 8 8 11 11 11 11 13/30 Read block 314 len 16384 14/30 Write block 421 len 16384 data: 7 6 6 6 15/30 Read block 595 len 8192 16/30 Read block 355 len 20480 17/30 Read block 78 len 32768 18/30 Write block 425 len 24576 data: 5 4 3 3 5 4 19/30 Write block 397 len 4096 data: 10 20/30 Read block 566 len 16384 21/30 Write block 498 len 4096 data: 6 22/30 Read block 226 len 40960 23/30 Read block 18 len 4096 24/30 Write block 146 len 12288 data: 7 7 9 25/30 Flush 26/30 Write block 563 len 40960 data: 7 7 7 8 7 7 9 8 8 12 27/30 Write block 458 len 12288 data: 9 12 12 28/30 Write block 102 len 4096 data: 6 29/30 Read block 259 len 20480 30/30 Read block 574 len 4096 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:03.789133849Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1555} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 146 146 146 F F F 1 020-039 91 91 91 146 146 146 F F F 2 040-059 91 91 91 146 146 146 F F F 3 060-079 92 92 92 148 148 148 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 87 87 87 139 139 139 T T T 6 120-139 87 87 87 139 139 139 F F F 7 140-159 92 92 92 148 148 148 F F F 8 160-179 85 85 85 137 137 137 F F F 9 180-199 92 92 92 148 148 148 F F F 10 200-219 88 88 88 141 141 141 F F F 11 220-239 87 87 87 139 139 139 F F F 12 240-259 91 91 91 146 146 146 F F F 13 260-279 86 86 86 138 138 138 F F F 14 280-299 91 91 91 146 146 146 F F F 15 300-319 89 89 89 143 143 143 F F F 16 320-339 90 90 90 144 144 144 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 88 88 88 142 142 142 F F F 19 380-399 92 92 92 148 148 148 F F F 20 400-419 92 92 92 147 147 147 F F F 21 420-439 92 92 92 148 148 148 F F F 22 440-459 92 92 92 148 148 148 T T T 23 460-479 92 92 92 148 148 148 T T T 24 480-499 92 92 92 148 148 148 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 90 90 90 144 144 144 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 91 91 91 146 146 146 T T T 29 580-599 84 84 84 134 134 134 F F F Max gen: 92, Max flush: 148 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 146 146 146 F F F 1 020-039 91 91 91 146 146 146 F F F 2 040-059 91 91 91 146 146 146 F F F 3 060-079 92 92 92 148 148 148 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 93 93 93 149 149 149 F F F 6 120-139 87 87 87 139 139 139 F F F 7 140-159 92 92 92 148 148 148 F F F 8 160-179 85 85 85 137 137 137 F F F 9 180-199 92 92 92 148 148 148 F F F 10 200-219 88 88 88 141 141 141 F F F 11 220-239 87 87 87 139 139 139 F F F 12 240-259 91 91 91 146 146 146 F F F 13 260-279 86 86 86 138 138 138 F F F 14 280-299 91 91 91 146 146 146 F F F 15 300-319 89 89 89 143 143 143 F F F 16 320-339 90 90 90 144 144 144 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 88 88 88 142 142 142 F F F 19 380-399 92 92 92 148 148 148 F F F 20 400-419 92 92 92 147 147 147 F F F 21 420-439 92 92 92 148 148 148 F F F 22 440-459 93 93 93 149 149 149 F F F 23 460-479 93 93 93 149 149 149 F F F 24 480-499 92 92 92 148 148 148 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 90 90 90 144 144 144 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 93 93 93 149 149 149 F F F 29 580-599 84 84 84 134 134 134 F F F Max gen: 93, Max flush: 149 Begin loop 47 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.062562428Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:08.104656576Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.105035772Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.13564464Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.136218124Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.136260449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.136276987Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.136294395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563} {"msg":"Crucible fb70676a-a6b3-415e-afe9-f4577a20ced3 has session id: c5490562-eb32-4f4b-b668-1f7bdd779c0e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.13676061Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Upstairs opts: Upstairs UUID: fb70676a-a6b3-415e-afe9-f4577a20ced3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.136805725Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Crucible stats registered with UUID: fb70676a-a6b3-415e-afe9-f4577a20ced3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.136831052Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166436308Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166477073Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563} {"msg":"fb70676a-a6b3-415e-afe9-f4577a20ced3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166513359Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166531187Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"0","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166560043Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"0","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166578521Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"1","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166597719Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"1","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166614537Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"2","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166645463Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"2","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166677399Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166710396Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166744412Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:53973)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166780207Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61850)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.166980514Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34046)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.167038167Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.167467887Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"2","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.167503803Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"1","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.167751874Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"0","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.168183914Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"2","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.16821358Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"1","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.168273893Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"0","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.168788513Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.1688161Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"[0]R flush_numbers[0..12]: [146, 146, 146, 148, 142, 149, 139, 148, 137, 148, 141, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.168879882Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"[0]R generation[0..12]: [91, 91, 91, 92, 88, 93, 87, 92, 85, 92, 88, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.1689013Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.168916098Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"[1]R flush_numbers[0..12]: [146, 146, 146, 148, 142, 149, 139, 148, 137, 148, 141, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.168930416Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"[1]R generation[0..12]: [91, 91, 91, 92, 88, 93, 87, 92, 85, 92, 88, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.168944285Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.168958453Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"[2]R flush_numbers[0..12]: [146, 146, 146, 148, 142, 149, 139, 148, 137, 148, 141, 139]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.168972662Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"[2]R generation[0..12]: [91, 91, 91, 92, 88, 93, 87, 92, 85, 92, 88, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169004848Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169022856Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169036274Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-13T15:55:08.169049753Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.16907458Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169088068Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"":"downstairs","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169101577Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169113875Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"0","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169127304Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"1","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169140492Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"client":"2","":"downstairs","session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169153861Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169166109Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"fb70676a-a6b3-415e-afe9-f4577a20ced3 is now active with session: c5490562-eb32-4f4b-b668-1f7bdd779c0e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169178967Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169191066Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563,"session_id":"c5490562-eb32-4f4b-b668-1f7bdd779c0e"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:08.169203375Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 308 len 12288 data: 5 5 6 02/30 Write block 344 len 20480 data: 9 9 9 7 8 03/30 Write block 226 len 28672 data: 5 5 8 8 10 10 10 04/30 Write block 278 len 8192 data: 7 7 05/30 Flush 06/30 Flush 07/30 Write block 266 len 8192 data: 12 11 08/30 Write block 122 len 40960 data: 7 11 10 10 12 13 11 7 4 4 09/30 Flush 10/30 Write block 74 len 8192 data: 6 5 11/30 Read block 493 len 28672 12/30 Write block 451 len 16384 data: 6 6 6 5 13/30 Read block 385 len 40960 14/30 Read block 568 len 40960 15/30 Write block 299 len 20480 data: 7 6 6 9 8 16/30 Read block 107 len 20480 17/30 Read block 586 len 28672 18/30 Write block 571 len 28672 data: 9 13 10 9 9 7 8 19/30 Write block 204 len 20480 data: 10 10 10 10 8 20/30 Read block 118 len 12288 21/30 Write block 7 len 4096 data: 10 22/30 Read block 25 len 24576 23/30 Read block 10 len 20480 24/30 Read block 79 len 40960 25/30 Read block 429 len 28672 26/30 Read block 205 len 8192 27/30 Write block 116 len 36864 data: 8 8 7 8 7 7 8 12 11 28/30 Read block 107 len 28672 29/30 Read block 445 len 32768 30/30 Read block 355 len 12288 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 2 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:08.189027102Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1563} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 146 146 146 T T T 1 020-039 91 91 91 146 146 146 F F F 2 040-059 91 91 91 146 146 146 F F F 3 060-079 92 92 92 148 148 148 T T T 4 080-099 88 88 88 142 142 142 F F F 5 100-119 93 93 93 149 149 149 T T T 6 120-139 94 94 94 152 152 152 T T T 7 140-159 92 92 92 148 148 148 F F F 8 160-179 85 85 85 137 137 137 F F F 9 180-199 92 92 92 148 148 148 F F F 10 200-219 88 88 88 141 141 141 T T T 11 220-239 94 94 94 150 150 150 F F F 12 240-259 91 91 91 146 146 146 F F F 13 260-279 94 94 94 152 152 152 F F F 14 280-299 91 91 91 146 146 146 T T T 15 300-319 94 94 94 150 150 150 T T T 16 320-339 90 90 90 144 144 144 F F F 17 340-359 94 94 94 150 150 150 F F F 18 360-379 88 88 88 142 142 142 F F F 19 380-399 92 92 92 148 148 148 F F F 20 400-419 92 92 92 147 147 147 F F F 21 420-439 92 92 92 148 148 148 F F F 22 440-459 93 93 93 149 149 149 T T T 23 460-479 93 93 93 149 149 149 F F F 24 480-499 92 92 92 148 148 148 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 90 90 90 144 144 144 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 93 93 93 149 149 149 T T T 29 580-599 84 84 84 134 134 134 F F F Max gen: 94, Max flush: 152 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 153 153 153 F F F 1 020-039 91 91 91 146 146 146 F F F 2 040-059 91 91 91 146 146 146 F F F 3 060-079 95 95 95 153 153 153 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 95 95 95 153 153 153 F F F 6 120-139 95 95 95 153 153 153 F F F 7 140-159 92 92 92 148 148 148 F F F 8 160-179 85 85 85 137 137 137 F F F 9 180-199 92 92 92 148 148 148 F F F 10 200-219 95 95 95 153 153 153 F F F 11 220-239 94 94 94 150 150 150 F F F 12 240-259 91 91 91 146 146 146 F F F 13 260-279 94 94 94 152 152 152 F F F 14 280-299 95 95 95 153 153 153 F F F 15 300-319 95 95 95 153 153 153 F F F 16 320-339 90 90 90 144 144 144 F F F 17 340-359 94 94 94 150 150 150 F F F 18 360-379 88 88 88 142 142 142 F F F 19 380-399 92 92 92 148 148 148 F F F 20 400-419 92 92 92 147 147 147 F F F 21 420-439 92 92 92 148 148 148 F F F 22 440-459 95 95 95 153 153 153 F F F 23 460-479 93 93 93 149 149 149 F F F 24 480-499 92 92 92 148 148 148 F F F 25 500-519 89 89 89 143 143 143 F F F 26 520-539 90 90 90 144 144 144 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 95 95 95 153 153 153 F F F 29 580-599 84 84 84 134 134 134 F F F Max gen: 95, Max flush: 153 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.519511562Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.564492883Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.565329775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.565369141Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.565389129Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.565404507Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571} {"msg":"Crucible 4b02b50d-95fe-4d0e-94c3-71f78b7984bd has session id: e0254d99-2532-4c4f-9fbd-731655f751fa","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.565883961Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Upstairs opts: Upstairs UUID: 4b02b50d-95fe-4d0e-94c3-71f78b7984bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.565933605Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Crucible stats registered with UUID: 4b02b50d-95fe-4d0e-94c3-71f78b7984bd","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.565953143Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596305551Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596357025Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571} {"msg":"4b02b50d-95fe-4d0e-94c3-71f78b7984bd active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596371823Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596385052Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"0","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596411559Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"0","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596426217Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"1","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596440395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"1","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596453924Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"2","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596468192Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"2","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.59649027Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596518016Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596544323Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:53614)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596682497Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:46973)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596730891Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:59577)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.596766267Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.59759705Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"1","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.597639896Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"0","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.597904574Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"2","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.59837373Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"1","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.598405416Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"0","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.598690493Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"2","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.59896968Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.598998047Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"[0]R flush_numbers[0..12]: [153, 146, 146, 153, 142, 153, 153, 148, 137, 148, 153, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599294023Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"[0]R generation[0..12]: [95, 91, 91, 95, 88, 95, 95, 92, 85, 92, 95, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599325279Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599340837Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"[1]R flush_numbers[0..12]: [153, 146, 146, 153, 142, 153, 153, 148, 137, 148, 153, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599355795Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"[1]R generation[0..12]: [95, 91, 91, 95, 88, 95, 95, 92, 85, 92, 95, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599370063Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599384212Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"[2]R flush_numbers[0..12]: [153, 146, 146, 153, 142, 153, 153, 148, 137, 148, 153, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.59939822Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"[2]R generation[0..12]: [95, 91, 91, 95, 88, 95, 95, 92, 85, 92, 95, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599412029Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599426437Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599439615Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599453234Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599466312Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"All extents match","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO ","level":30,"time":"2025-01-13T15:55:12.59948641Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"":"downstairs","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599500048Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599512137Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"0","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599528315Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"1","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599541883Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"client":"2","":"downstairs","session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599555042Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599567291Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"4b02b50d-95fe-4d0e-94c3-71f78b7984bd is now active with session: e0254d99-2532-4c4f-9fbd-731655f751fa","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599580139Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599592128Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571,"session_id":"e0254d99-2532-4c4f-9fbd-731655f751fa"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:12.599604306Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 224 len 4096 data: 7 03/30 Flush 04/30 Write block 402 len 12288 data: 9 10 8 05/30 Read block 264 len 8192 06/30 Write block 514 len 4096 data: 6 07/30 Read block 270 len 4096 08/30 Write block 255 len 36864 data: 10 7 8 9 9 10 10 9 9 09/30 Flush 10/30 Write block 160 len 24576 data: 11 13 14 12 12 10 11/30 Read block 527 len 4096 12/30 Flush 13/30 Write block 472 len 40960 data: 5 4 6 6 7 6 9 11 10 8 14/30 Read block 152 len 12288 15/30 Read block 430 len 4096 16/30 Write block 325 len 16384 data: 9 7 6 7 17/30 Read block 2 len 28672 18/30 Read block 171 len 32768 19/30 Read block 53 len 12288 20/30 Write block 54 len 16384 data: 10 10 7 7 21/30 Write block 508 len 32768 data: 7 7 7 7 7 6 7 8 22/30 Read block 452 len 28672 23/30 Write block 594 len 4096 data: 10 24/30 Flush 25/30 Write block 200 len 4096 data: 9 26/30 Read block 572 len 16384 27/30 Write block 3 len 4096 data: 3 28/30 Read block 521 len 12288 29/30 Read block 422 len 20480 30/30 Read block 204 len 32768 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 5 6 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:12.618810755Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1571} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 153 153 153 T T T 1 020-039 91 91 91 146 146 146 F F F 2 040-059 96 96 96 158 158 158 F F F 3 060-079 95 95 95 153 153 153 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 95 95 95 153 153 153 F F F 6 120-139 95 95 95 153 153 153 F F F 7 140-159 92 92 92 148 148 148 F F F 8 160-179 96 96 96 157 157 157 F F F 9 180-199 92 92 92 148 148 148 F F F 10 200-219 95 95 95 153 153 153 T T T 11 220-239 96 96 96 155 155 155 F F F 12 240-259 96 96 96 156 156 156 F F F 13 260-279 96 96 96 156 156 156 F F F 14 280-299 95 95 95 153 153 153 F F F 15 300-319 95 95 95 153 153 153 F F F 16 320-339 96 96 96 158 158 158 F F F 17 340-359 94 94 94 150 150 150 F F F 18 360-379 88 88 88 142 142 142 F F F 19 380-399 92 92 92 148 148 148 F F F 20 400-419 96 96 96 156 156 156 F F F 21 420-439 92 92 92 148 148 148 F F F 22 440-459 95 95 95 153 153 153 F F F 23 460-479 96 96 96 158 158 158 F F F 24 480-499 96 96 96 158 158 158 F F F 25 500-519 96 96 96 158 158 158 F F F 26 520-539 90 90 90 144 144 144 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 95 95 95 153 153 153 F F F 29 580-599 96 96 96 158 158 158 F F F Max gen: 96, Max flush: 158 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 159 159 159 F F F 1 020-039 91 91 91 146 146 146 F F F 2 040-059 96 96 96 158 158 158 F F F 3 060-079 95 95 95 153 153 153 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 95 95 95 153 153 153 F F F 6 120-139 95 95 95 153 153 153 F F F 7 140-159 92 92 92 148 148 148 F F F 8 160-179 96 96 96 157 157 157 F F F 9 180-199 92 92 92 148 148 148 F F F 10 200-219 97 97 97 159 159 159 F F F 11 220-239 96 96 96 155 155 155 F F F 12 240-259 96 96 96 156 156 156 F F F 13 260-279 96 96 96 156 156 156 F F F 14 280-299 95 95 95 153 153 153 F F F 15 300-319 95 95 95 153 153 153 F F F 16 320-339 96 96 96 158 158 158 F F F 17 340-359 94 94 94 150 150 150 F F F 18 360-379 88 88 88 142 142 142 F F F 19 380-399 92 92 92 148 148 148 F F F 20 400-419 96 96 96 156 156 156 F F F 21 420-439 92 92 92 148 148 148 F F F 22 440-459 95 95 95 153 153 153 F F F 23 460-479 96 96 96 158 158 158 F F F 24 480-499 96 96 96 158 158 158 F F F 25 500-519 96 96 96 158 158 158 F F F 26 520-539 90 90 90 144 144 144 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 95 95 95 153 153 153 F F F 29 580-599 96 96 96 158 158 158 F F F Max gen: 97, Max flush: 159 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:16.895399304Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:16.938420164Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:16.938807699Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:16.969657539Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:16.970361257Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:16.970407462Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:16.970425159Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:16.970443297Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579} {"msg":"Crucible 9bb14104-bf97-4865-bf82-e20f943010a7 has session id: 9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:16.970884416Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Upstairs opts: Upstairs UUID: 9bb14104-bf97-4865-bf82-e20f943010a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:16.970940099Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Crucible stats registered with UUID: 9bb14104-bf97-4865-bf82-e20f943010a7","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:16.970975265Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001406484Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001492154Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579} {"msg":"9bb14104-bf97-4865-bf82-e20f943010a7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001603461Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001635298Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"0","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001696671Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"0","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001738766Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"1","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001774581Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"1","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001804458Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"2","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001834914Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"2","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001861102Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001893528Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001933173Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:39275)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.001971589Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34942)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.002005845Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43709)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.002030862Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.002278103Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"2","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.00247194Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"1","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.002548611Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"0","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.00299151Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"1","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003031745Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"2","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003130033Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"0","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.00358912Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003700277Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"[0]R flush_numbers[0..12]: [159, 146, 158, 153, 142, 153, 153, 148, 157, 148, 159, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003798465Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"[0]R generation[0..12]: [97, 91, 96, 95, 88, 95, 95, 92, 96, 92, 97, 96]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003819313Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003834831Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"[1]R flush_numbers[0..12]: [159, 146, 158, 153, 142, 153, 153, 148, 157, 148, 159, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003849179Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"[1]R generation[0..12]: [97, 91, 96, 95, 88, 95, 95, 92, 96, 92, 97, 96]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003863418Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003877796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"[2]R flush_numbers[0..12]: [159, 146, 158, 153, 142, 153, 153, 148, 157, 148, 159, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003891594Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"[2]R generation[0..12]: [97, 91, 96, 95, 88, 95, 95, 92, 96, 92, 97, 96]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003905283Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003919231Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.00393234Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003949678Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.003963256Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:55:17.003976215Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"":"downstairs","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.004020039Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.004034938Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"0","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.004048716Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"1","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.004062125Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"client":"2","":"downstairs","session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.004075553Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.004087882Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"9bb14104-bf97-4865-bf82-e20f943010a7 is now active with session: 9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.00410091Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.004113089Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579,"session_id":"9d6cbcdd-582c-4f6d-83fc-d859efe3ba7d"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:17.004125467Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 104 len 36864 data: 8 9 6 7 7 5 5 4 4 02/30 Write block 389 len 4096 data: 9 03/30 Read block 472 len 32768 04/30 Write block 348 len 4096 data: 9 05/30 Write block 8 len 32768 data: 10 8 8 8 9 9 8 7 06/30 Read block 533 len 32768 07/30 Write block 521 len 4096 data: 7 08/30 Write block 112 len 36864 data: 5 3 4 5 9 9 8 9 8 09/30 Read block 317 len 28672 10/30 Write block 446 len 40960 data: 5 4 4 6 6 7 7 7 6 8 11/30 Read block 280 len 24576 12/30 Flush 13/30 Write block 477 len 8192 data: 7 10 14/30 Read block 445 len 20480 15/30 Write block 242 len 8192 data: 7 5 16/30 Read block 455 len 32768 17/30 Read block 557 len 20480 18/30 Write block 289 len 24576 data: 8 9 10 12 10 10 19/30 Write block 219 len 24576 data: 8 8 8 7 7 8 20/30 Read block 37 len 20480 21/30 Read block 269 len 16384 22/30 Read block 556 len 24576 23/30 Read block 501 len 16384 24/30 Write block 290 len 8192 data: 10 11 25/30 Read block 102 len 40960 26/30 Write block 362 len 24576 data: 7 7 7 7 7 8 27/30 Write block 27 len 16384 data: 8 7 7 6 28/30 Write block 461 len 8192 data: 12 12 29/30 Read block 251 len 4096 30/30 Read block 354 len 36864 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 2 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:17.023356913Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1579} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 160 160 160 F F F 1 020-039 91 91 91 146 146 146 T T T 2 040-059 96 96 96 158 158 158 F F F 3 060-079 95 95 95 153 153 153 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 98 98 98 160 160 160 F F F 6 120-139 98 98 98 160 160 160 F F F 7 140-159 92 92 92 148 148 148 F F F 8 160-179 96 96 96 157 157 157 F F F 9 180-199 92 92 92 148 148 148 F F F 10 200-219 97 97 97 159 159 159 T T T 11 220-239 96 96 96 155 155 155 T T T 12 240-259 96 96 96 156 156 156 T T T 13 260-279 96 96 96 156 156 156 F F F 14 280-299 95 95 95 153 153 153 T T T 15 300-319 95 95 95 153 153 153 F F F 16 320-339 96 96 96 158 158 158 F F F 17 340-359 98 98 98 160 160 160 F F F 18 360-379 88 88 88 142 142 142 T T T 19 380-399 98 98 98 160 160 160 F F F 20 400-419 96 96 96 156 156 156 F F F 21 420-439 92 92 92 148 148 148 F F F 22 440-459 98 98 98 160 160 160 F F F 23 460-479 96 96 96 158 158 158 T T T 24 480-499 96 96 96 158 158 158 F F F 25 500-519 96 96 96 158 158 158 F F F 26 520-539 98 98 98 160 160 160 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 95 95 95 153 153 153 F F F 29 580-599 96 96 96 158 158 158 F F F Max gen: 98, Max flush: 160 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 160 160 160 F F F 1 020-039 99 99 99 161 161 161 F F F 2 040-059 96 96 96 158 158 158 F F F 3 060-079 95 95 95 153 153 153 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 98 98 98 160 160 160 F F F 6 120-139 98 98 98 160 160 160 F F F 7 140-159 92 92 92 148 148 148 F F F 8 160-179 96 96 96 157 157 157 F F F 9 180-199 92 92 92 148 148 148 F F F 10 200-219 99 99 99 161 161 161 F F F 11 220-239 99 99 99 161 161 161 F F F 12 240-259 99 99 99 161 161 161 F F F 13 260-279 96 96 96 156 156 156 F F F 14 280-299 99 99 99 161 161 161 F F F 15 300-319 95 95 95 153 153 153 F F F 16 320-339 96 96 96 158 158 158 F F F 17 340-359 98 98 98 160 160 160 F F F 18 360-379 99 99 99 161 161 161 F F F 19 380-399 98 98 98 160 160 160 F F F 20 400-419 96 96 96 156 156 156 F F F 21 420-439 92 92 92 148 148 148 F F F 22 440-459 98 98 98 160 160 160 F F F 23 460-479 99 99 99 161 161 161 F F F 24 480-499 96 96 96 158 158 158 F F F 25 500-519 96 96 96 158 158 158 F F F 26 520-539 98 98 98 160 160 160 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 95 95 95 153 153 153 F F F 29 580-599 96 96 96 158 158 158 F F F Max gen: 99, Max flush: 161 Begin loop 50 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.343141154Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.388589491Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.389545739Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.389586404Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.389600803Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.389615091Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587} {"msg":"Crucible 43ebee02-3ad2-4e29-a2ea-53d5e6d521b7 has session id: b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.39005556Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Upstairs opts: Upstairs UUID: 43ebee02-3ad2-4e29-a2ea-53d5e6d521b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.390100464Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Crucible stats registered with UUID: 43ebee02-3ad2-4e29-a2ea-53d5e6d521b7","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.390117132Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.422398526Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.422442941Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587} {"msg":"43ebee02-3ad2-4e29-a2ea-53d5e6d521b7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.422465908Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.422481706Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"0","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.422511342Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"0","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.42253018Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"1","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.422548008Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"1","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.422622629Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"2","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.422669314Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"2","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.4227063Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.422740046Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.422772702Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41325)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.422806778Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59925)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.423036101Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57388)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.423090385Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.423621813Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"0","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.423670357Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"2","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.423835308Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"1","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.424350038Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"0","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.424381054Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"2","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.424409591Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"1","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.424863288Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.424966476Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"[0]R flush_numbers[0..12]: [160, 161, 158, 153, 142, 160, 160, 148, 157, 148, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425044227Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"[0]R generation[0..12]: [98, 99, 96, 95, 88, 98, 98, 92, 96, 92, 99, 99]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425066324Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425081022Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"[1]R flush_numbers[0..12]: [160, 161, 158, 153, 142, 160, 160, 148, 157, 148, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425095111Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"[1]R generation[0..12]: [98, 99, 96, 95, 88, 98, 98, 92, 96, 92, 99, 99]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425112599Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425127007Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"[2]R flush_numbers[0..12]: [160, 161, 158, 153, 142, 160, 160, 148, 157, 148, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425141275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"[2]R generation[0..12]: [98, 99, 96, 95, 88, 98, 98, 92, 96, 92, 99, 99]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425154964Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425169052Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425182241Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:55:21.425195469Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425213207Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425226446Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"":"downstairs","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425239834Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425251983Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"0","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425265441Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"1","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425278559Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"client":"2","":"downstairs","session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425291948Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425304327Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"43ebee02-3ad2-4e29-a2ea-53d5e6d521b7 is now active with session: b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425322484Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425335823Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587,"session_id":"b62d4679-c1da-4d82-aaae-4ca4cd6b1a2f"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:21.425348471Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 514 len 20480 data: 8 9 12 11 11 02/30 Read block 217 len 4096 03/30 Read block 48 len 32768 04/30 Read block 413 len 40960 05/30 Read block 475 len 36864 06/30 Read block 131 len 8192 07/30 Read block 248 len 32768 08/30 Write block 307 len 4096 data: 5 09/30 Read block 330 len 16384 10/30 Write block 565 len 40960 data: 8 9 8 8 10 9 10 14 11 10 11/30 Write block 181 len 8192 data: 8 7 12/30 Flush 13/30 Flush 14/30 Flush 15/30 Write block 21 len 40960 data: 10 10 11 11 10 10 9 8 8 7 16/30 Write block 298 len 36864 data: 7 8 7 7 10 9 5 5 5 17/30 Read block 214 len 40960 18/30 Read block 278 len 4096 19/30 Read block 19 len 24576 20/30 Read block 94 len 36864 21/30 Flush 22/30 Write block 427 len 24576 data: 4 4 6 5 5 8 23/30 Flush 24/30 Write block 209 len 32768 data: 9 8 11 12 12 11 11 11 25/30 Read block 494 len 8192 26/30 Read block 140 len 16384 27/30 Write block 65 len 36864 data: 5 8 10 10 9 9 8 6 6 28/30 Write block 52 len 4096 data: 10 29/30 Write block 195 len 40960 data: 9 8 8 9 10 10 8 7 9 11 30/30 Write block 193 len 20480 data: 8 9 10 9 9 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:21.445578891Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1587} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 160 160 160 F F F 1 020-039 100 100 100 165 165 165 F F F 2 040-059 96 96 96 158 158 158 T T T 3 060-079 95 95 95 153 153 153 T T T 4 080-099 88 88 88 142 142 142 F F F 5 100-119 98 98 98 160 160 160 F F F 6 120-139 98 98 98 160 160 160 F F F 7 140-159 92 92 92 148 148 148 F F F 8 160-179 96 96 96 157 157 157 F F F 9 180-199 100 100 100 162 162 162 T T T 10 200-219 99 99 99 161 161 161 T T T 11 220-239 99 99 99 161 161 161 F F F 12 240-259 99 99 99 161 161 161 F F F 13 260-279 96 96 96 156 156 156 F F F 14 280-299 100 100 100 165 165 165 F F F 15 300-319 100 100 100 165 165 165 F F F 16 320-339 96 96 96 158 158 158 F F F 17 340-359 98 98 98 160 160 160 F F F 18 360-379 99 99 99 161 161 161 F F F 19 380-399 98 98 98 160 160 160 F F F 20 400-419 96 96 96 156 156 156 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 98 98 98 160 160 160 F F F 23 460-479 99 99 99 161 161 161 F F F 24 480-499 96 96 96 158 158 158 F F F 25 500-519 100 100 100 162 162 162 F F F 26 520-539 98 98 98 160 160 160 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 100 100 100 162 162 162 F F F 29 580-599 96 96 96 158 158 158 F F F Max gen: 100, Max flush: 166 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 160 160 160 F F F 1 020-039 100 100 100 165 165 165 F F F 2 040-059 101 101 101 167 167 167 F F F 3 060-079 101 101 101 167 167 167 F F F 4 080-099 88 88 88 142 142 142 F F F 5 100-119 98 98 98 160 160 160 F F F 6 120-139 98 98 98 160 160 160 F F F 7 140-159 92 92 92 148 148 148 F F F 8 160-179 96 96 96 157 157 157 F F F 9 180-199 101 101 101 167 167 167 F F F 10 200-219 101 101 101 167 167 167 F F F 11 220-239 99 99 99 161 161 161 F F F 12 240-259 99 99 99 161 161 161 F F F 13 260-279 96 96 96 156 156 156 F F F 14 280-299 100 100 100 165 165 165 F F F 15 300-319 100 100 100 165 165 165 F F F 16 320-339 96 96 96 158 158 158 F F F 17 340-359 98 98 98 160 160 160 F F F 18 360-379 99 99 99 161 161 161 F F F 19 380-399 98 98 98 160 160 160 F F F 20 400-419 96 96 96 156 156 156 F F F 21 420-439 100 100 100 166 166 166 F F F 22 440-459 98 98 98 160 160 160 F F F 23 460-479 99 99 99 161 161 161 F F F 24 480-499 96 96 96 158 158 158 F F F 25 500-519 100 100 100 162 162 162 F F F 26 520-539 98 98 98 160 160 160 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 100 100 100 162 162 162 F F F 29 580-599 96 96 96 158 158 158 F F F Max gen: 101, Max flush: 167 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.715506538Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.760125271Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.760853906Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.760891082Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.76090957Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.760934247Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595} {"msg":"Crucible da8514ed-45fc-4436-98c1-64221d5e4ec5 has session id: 4af433b5-a094-403e-a88c-80d98e2d3d3f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.761462315Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Upstairs opts: Upstairs UUID: da8514ed-45fc-4436-98c1-64221d5e4ec5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.76151079Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Crucible stats registered with UUID: da8514ed-45fc-4436-98c1-64221d5e4ec5","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.761530747Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.791704236Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.791736812Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595} {"msg":"da8514ed-45fc-4436-98c1-64221d5e4ec5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.791785376Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.791804444Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"0","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.791830841Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"0","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.791848159Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"1","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.791865837Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"1","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.791882435Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"2","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.791900193Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"2","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.791932079Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.791965865Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.791998001Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60808)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.792031677Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60561)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.792061554Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42831)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.79209561Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.792737885Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"2","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.792930483Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"1","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.793022742Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"0","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.793524953Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"2","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.79355747Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"0","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.793794682Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"1","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794059441Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794147961Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"[0]R flush_numbers[0..12]: [160, 165, 167, 167, 142, 160, 160, 148, 157, 167, 167, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794382353Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"[0]R generation[0..12]: [98, 100, 101, 101, 88, 98, 98, 92, 96, 101, 101, 99]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794416849Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794435947Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"[1]R flush_numbers[0..12]: [160, 165, 167, 167, 142, 160, 160, 148, 157, 167, 167, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794453775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"[1]R generation[0..12]: [98, 100, 101, 101, 88, 98, 98, 92, 96, 101, 101, 99]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794470993Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794488731Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"[2]R flush_numbers[0..12]: [160, 165, 167, 167, 142, 160, 160, 148, 157, 167, 167, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794506039Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"[2]R generation[0..12]: [98, 100, 101, 101, 88, 98, 98, 92, 96, 101, 101, 99]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:55:25.794523197Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794546364Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794562892Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.79457959Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794595968Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794612116Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"":"downstairs","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794628494Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794643273Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"0","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794662031Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"1","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794678619Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"client":"2","":"downstairs","session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-13T15:55:25.794695217Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794714404Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"da8514ed-45fc-4436-98c1-64221d5e4ec5 is now active with session: 4af433b5-a094-403e-a88c-80d98e2d3d3f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794730603Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794745831Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595,"session_id":"4af433b5-a094-403e-a88c-80d98e2d3d3f"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:25.794761119Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 320 len 36864 data: 7 7 8 8 7 10 8 7 8 02/30 Write block 259 len 28672 data: 10 11 11 10 10 10 10 03/30 Write block 529 len 20480 data: 7 9 13 13 11 04/30 Write block 185 len 8192 data: 7 8 05/30 Write block 518 len 36864 data: 12 12 9 8 7 6 6 3 3 06/30 Read block 133 len 20480 07/30 Read block 582 len 28672 08/30 Read block 370 len 36864 09/30 Write block 226 len 12288 data: 6 6 9 10/30 Read block 516 len 28672 11/30 Read block 174 len 16384 12/30 Read block 161 len 8192 13/30 Read block 254 len 12288 14/30 Write block 20 len 28672 data: 10 11 11 12 12 11 11 15/30 Read block 361 len 8192 16/30 Write block 426 len 4096 data: 5 17/30 Write block 244 len 4096 data: 5 18/30 Flush 19/30 Flush 20/30 Flush 21/30 Write block 148 len 24576 data: 10 10 9 10 9 9 22/30 Write block 197 len 24576 data: 10 10 11 11 9 8 23/30 Read block 432 len 32768 24/30 Read block 433 len 4096 25/30 Write block 565 len 40960 data: 9 10 9 9 11 10 11 15 12 11 26/30 Write block 226 len 28672 data: 7 7 10 9 11 11 11 27/30 Read block 125 len 36864 28/30 Read block 148 len 16384 29/30 Write block 95 len 20480 data: 10 9 8 8 6 30/30 Write block 457 len 12288 data: 9 10 13 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:25.813475665Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1595} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 160 160 160 F F F 1 020-039 102 102 102 168 168 168 F F F 2 040-059 101 101 101 167 167 167 F F F 3 060-079 101 101 101 167 167 167 F F F 4 080-099 88 88 88 142 142 142 T T T 5 100-119 98 98 98 160 160 160 F F F 6 120-139 98 98 98 160 160 160 F F F 7 140-159 92 92 92 148 148 148 T T T 8 160-179 96 96 96 157 157 157 F F F 9 180-199 102 102 102 168 168 168 T T T 10 200-219 101 101 101 167 167 167 T T T 11 220-239 102 102 102 168 168 168 T T T 12 240-259 102 102 102 168 168 168 F F F 13 260-279 102 102 102 168 168 168 F F F 14 280-299 100 100 100 165 165 165 F F F 15 300-319 100 100 100 165 165 165 F F F 16 320-339 102 102 102 168 168 168 F F F 17 340-359 98 98 98 160 160 160 F F F 18 360-379 99 99 99 161 161 161 F F F 19 380-399 98 98 98 160 160 160 F F F 20 400-419 96 96 96 156 156 156 F F F 21 420-439 102 102 102 168 168 168 F F F 22 440-459 98 98 98 160 160 160 T T T 23 460-479 99 99 99 161 161 161 F F F 24 480-499 96 96 96 158 158 158 F F F 25 500-519 102 102 102 168 168 168 F F F 26 520-539 102 102 102 168 168 168 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 100 100 100 162 162 162 T T T 29 580-599 96 96 96 158 158 158 F F F Max gen: 102, Max flush: 168 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 160 160 160 F F F 1 020-039 102 102 102 168 168 168 F F F 2 040-059 101 101 101 167 167 167 F F F 3 060-079 101 101 101 167 167 167 F F F 4 080-099 103 103 103 169 169 169 F F F 5 100-119 98 98 98 160 160 160 F F F 6 120-139 98 98 98 160 160 160 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 96 96 96 157 157 157 F F F 9 180-199 103 103 103 169 169 169 F F F 10 200-219 103 103 103 169 169 169 F F F 11 220-239 103 103 103 169 169 169 F F F 12 240-259 102 102 102 168 168 168 F F F 13 260-279 102 102 102 168 168 168 F F F 14 280-299 100 100 100 165 165 165 F F F 15 300-319 100 100 100 165 165 165 F F F 16 320-339 102 102 102 168 168 168 F F F 17 340-359 98 98 98 160 160 160 F F F 18 360-379 99 99 99 161 161 161 F F F 19 380-399 98 98 98 160 160 160 F F F 20 400-419 96 96 96 156 156 156 F F F 21 420-439 102 102 102 168 168 168 F F F 22 440-459 103 103 103 169 169 169 F F F 23 460-479 99 99 99 161 161 161 F F F 24 480-499 96 96 96 158 158 158 F F F 25 500-519 102 102 102 168 168 168 F F F 26 520-539 102 102 102 168 168 168 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 103 103 103 169 169 169 F F F 29 580-599 96 96 96 158 158 158 F F F Max gen: 103, Max flush: 169 Begin loop 52 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.101298194Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.146506528Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.147244752Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.147285177Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.147301766Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.147319353Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603} {"msg":"Crucible b6bab53e-be97-4fab-a435-fa5bf91feaa1 has session id: 3d4ef29d-e993-4181-ba93-fbbf4f15de5a","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.147795238Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Upstairs opts: Upstairs UUID: b6bab53e-be97-4fab-a435-fa5bf91feaa1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.147832374Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Crucible stats registered with UUID: b6bab53e-be97-4fab-a435-fa5bf91feaa1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.147849972Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.178067706Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.17811704Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603} {"msg":"b6bab53e-be97-4fab-a435-fa5bf91feaa1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.178133068Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.178149076Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"0","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.178176563Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"0","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.17820062Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"1","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.178219268Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"1","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.178236116Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"2","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.178254094Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"2","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.178270782Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.17828825Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.178305228Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"ds_connection connected from Ok(127.0.0.1:65417)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.178679534Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44806)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.178725589Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56347)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.178758435Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.179299512Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"1","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.179343047Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"0","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.179493719Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"2","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.179938677Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"1","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.179983992Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"0","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180219894Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"2","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180484134Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180569594Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"[0]R flush_numbers[0..12]: [160, 168, 167, 167, 169, 160, 160, 169, 157, 169, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180738784Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"[0]R generation[0..12]: [98, 102, 101, 101, 103, 98, 98, 103, 96, 103, 103, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180759631Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.18077461Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"[1]R flush_numbers[0..12]: [160, 168, 167, 167, 169, 160, 160, 169, 157, 169, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180789538Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"[1]R generation[0..12]: [98, 102, 101, 101, 103, 98, 98, 103, 96, 103, 103, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180803737Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180818215Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"[2]R flush_numbers[0..12]: [160, 168, 167, 167, 169, 160, 160, 169, 157, 169, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180832473Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"[2]R generation[0..12]: [98, 102, 101, 101, 103, 98, 98, 103, 96, 103, 103, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180846292Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.1808603Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180877278Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180891136Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180904305Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:55:30.180917243Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"":"downstairs","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.1809421Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180966638Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"0","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.180994164Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"1","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.181020571Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"client":"2","":"downstairs","session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.181048108Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.181074215Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"b6bab53e-be97-4fab-a435-fa5bf91feaa1 is now active with session: 3d4ef29d-e993-4181-ba93-fbbf4f15de5a","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.181101202Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.181126429Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603,"session_id":"3d4ef29d-e993-4181-ba93-fbbf4f15de5a"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:30.181154905Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 349 len 16384 data: 7 8 6 5 02/30 Read block 524 len 24576 03/30 Write block 414 len 4096 data: 11 04/30 Write block 346 len 8192 data: 10 8 05/30 Read block 269 len 16384 06/30 Flush 07/30 Write block 443 len 16384 data: 7 7 5 6 08/30 Flush 09/30 Write block 457 len 28672 data: 10 11 14 13 13 13 12 10/30 Write block 176 len 36864 data: 5 8 8 8 9 9 8 8 8 11/30 Write block 27 len 12288 data: 10 9 9 12/30 Read block 314 len 28672 13/30 Write block 19 len 36864 data: 13 11 12 12 13 13 12 12 11 14/30 Write block 286 len 8192 data: 5 6 15/30 Write block 208 len 4096 data: 9 16/30 Write block 389 len 4096 data: 10 17/30 Read block 575 len 40960 18/30 Flush 19/30 Write block 217 len 24576 data: 9 11 9 9 9 8 20/30 Read block 425 len 20480 21/30 Write block 16 len 4096 data: 11 22/30 Write block 105 len 16384 data: 10 7 8 8 23/30 Write block 26 len 12288 data: 13 12 10 24/30 Read block 3 len 12288 25/30 Write block 191 len 24576 data: 10 7 9 10 11 10 26/30 Write block 327 len 8192 data: 8 9 27/30 Write block 496 len 36864 data: 6 5 7 10 13 13 12 12 11 28/30 Write block 371 len 12288 data: 6 6 7 29/30 Read block 174 len 12288 30/30 Read block 27 len 20480 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 6 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:30.199122379Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1603} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 172 172 172 T T T 1 020-039 104 104 104 172 172 172 T T T 2 040-059 101 101 101 167 167 167 F F F 3 060-079 101 101 101 167 167 167 F F F 4 080-099 103 103 103 169 169 169 F F F 5 100-119 98 98 98 160 160 160 T T T 6 120-139 98 98 98 160 160 160 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 104 104 104 172 172 172 F F F 9 180-199 104 104 104 172 172 172 T T T 10 200-219 104 104 104 172 172 172 T T T 11 220-239 103 103 103 169 169 169 T T T 12 240-259 102 102 102 168 168 168 F F F 13 260-279 102 102 102 168 168 168 F F F 14 280-299 104 104 104 172 172 172 F F F 15 300-319 100 100 100 165 165 165 F F F 16 320-339 102 102 102 168 168 168 T T T 17 340-359 104 104 104 170 170 170 F F F 18 360-379 99 99 99 161 161 161 T T T 19 380-399 104 104 104 172 172 172 F F F 20 400-419 104 104 104 170 170 170 F F F 21 420-439 102 102 102 168 168 168 F F F 22 440-459 104 104 104 172 172 172 F F F 23 460-479 104 104 104 172 172 172 F F F 24 480-499 96 96 96 158 158 158 T T T 25 500-519 102 102 102 168 168 168 T T T 26 520-539 102 102 102 168 168 168 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 103 103 103 169 169 169 F F F 29 580-599 96 96 96 158 158 158 F F F Max gen: 104, Max flush: 172 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 173 173 173 F F F 1 020-039 105 105 105 173 173 173 F F F 2 040-059 101 101 101 167 167 167 F F F 3 060-079 101 101 101 167 167 167 F F F 4 080-099 103 103 103 169 169 169 F F F 5 100-119 105 105 105 173 173 173 F F F 6 120-139 98 98 98 160 160 160 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 104 104 104 172 172 172 F F F 9 180-199 105 105 105 173 173 173 F F F 10 200-219 105 105 105 173 173 173 F F F 11 220-239 105 105 105 173 173 173 F F F 12 240-259 102 102 102 168 168 168 F F F 13 260-279 102 102 102 168 168 168 F F F 14 280-299 104 104 104 172 172 172 F F F 15 300-319 100 100 100 165 165 165 F F F 16 320-339 105 105 105 173 173 173 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 105 105 105 173 173 173 F F F 19 380-399 104 104 104 172 172 172 F F F 20 400-419 104 104 104 170 170 170 F F F 21 420-439 102 102 102 168 168 168 F F F 22 440-459 104 104 104 172 172 172 F F F 23 460-479 104 104 104 172 172 172 F F F 24 480-499 105 105 105 173 173 173 F F F 25 500-519 105 105 105 173 173 173 F F F 26 520-539 102 102 102 168 168 168 F F F 27 540-559 91 91 91 146 146 146 F F F 28 560-579 103 103 103 169 169 169 F F F 29 580-599 96 96 96 158 158 158 F F F Max gen: 105, Max flush: 173 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.497567819Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.542707732Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.543490451Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.543542205Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.543567791Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.543593359Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611} {"msg":"Crucible 6551bb1c-5b55-4878-9df3-6b26849d8651 has session id: a2317272-0287-400d-8a87-427950727cf2","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.544042016Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Upstairs opts: Upstairs UUID: 6551bb1c-5b55-4878-9df3-6b26849d8651, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.544101189Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Crucible stats registered with UUID: 6551bb1c-5b55-4878-9df3-6b26849d8651","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.544130126Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.573987262Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.574050085Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611} {"msg":"6551bb1c-5b55-4878-9df3-6b26849d8651 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.574080241Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.574098959Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"0","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.574139244Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"0","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.57417399Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"1","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.574206396Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"1","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.574233833Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"2","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.57426292Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"2","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.574288077Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.574311234Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.574335541Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"ds_connection connected from Ok(127.0.0.1:48368)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.574362028Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"ds_connection connected from Ok(127.0.0.1:65100)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.574388845Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"ds_connection connected from Ok(127.0.0.1:49412)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.574419731Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.575175143Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"1","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.575217449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"0","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.575310958Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"2","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.575902358Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"0","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.575932525Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"2","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.575948303Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"1","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.57648756Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576525666Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"[0]R flush_numbers[0..12]: [173, 173, 167, 167, 169, 173, 160, 169, 172, 173, 173, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576549143Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"[0]R generation[0..12]: [105, 105, 101, 101, 103, 105, 98, 103, 104, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576565631Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.57658035Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"[1]R flush_numbers[0..12]: [173, 173, 167, 167, 169, 173, 160, 169, 172, 173, 173, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576594588Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"[1]R generation[0..12]: [105, 105, 101, 101, 103, 105, 98, 103, 104, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576608886Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576627544Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"[2]R flush_numbers[0..12]: [173, 173, 167, 167, 169, 173, 160, 169, 172, 173, 173, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576642102Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"[2]R generation[0..12]: [105, 105, 101, 101, 103, 105, 98, 103, 104, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576656001Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576670139Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576683227Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576697846Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576721603Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"Wait for a query_work_queue command to finish before sending IO } {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576753309Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"":"downstairs","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576775937Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576788645Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"0","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576802014Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"1","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576815152Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"client":"2","":"downstairs","session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576828301Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-13T15:55:34.576840579Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"6551bb1c-5b55-4878-9df3-6b26849d8651 is now active with session: a2317272-0287-400d-8a87-427950727cf2","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576854427Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576869286Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611,"session_id":"a2317272-0287-400d-8a87-427950727cf2"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:34.576881874Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 537 len 40960 data: 12 11 11 10 9 7 7 7 8 9 03/30 Read block 29 len 20480 04/30 Read block 8 len 8192 05/30 Flush 06/30 Write block 31 len 12288 data: 7 5 5 07/30 Read block 102 len 36864 08/30 Write block 54 len 24576 data: 11 11 8 8 7 7 09/30 Write block 490 len 20480 data: 7 6 6 4 6 10/30 Write block 116 len 28672 data: 10 10 9 10 9 8 9 11/30 Read block 299 len 12288 12/30 Write block 284 len 16384 data: 4 5 6 7 13/30 Flush 14/30 Write block 109 len 24576 data: 6 6 5 6 4 5 15/30 Read block 294 len 40960 16/30 Write block 433 len 28672 data: 7 6 6 7 10 9 7 17/30 Read block 158 len 24576 18/30 Write block 579 len 32768 data: 5 7 8 7 9 7 8 6 19/30 Write block 413 len 32768 data: 13 12 10 9 7 6 7 7 20/30 Write block 593 len 20480 data: 11 11 8 7 5 21/30 Flush 22/30 Write block 379 len 16384 data: 6 7 9 10 23/30 Write block 546 len 20480 data: 10 8 6 6 7 24/30 Read block 94 len 36864 25/30 Write block 531 len 36864 data: 14 14 12 12 10 10 13 12 12 26/30 Write block 116 len 28672 data: 11 11 10 11 10 9 10 27/30 Read block 236 len 40960 28/30 Write block 78 len 28672 data: 4 3 4 4 9 9 8 29/30 Write block 180 len 40960 data: 10 10 9 9 9 8 9 7 8 10 30/30 Read block 541 len 32768 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:34.599343783Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1611} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 173 173 173 F F F 1 020-039 106 106 106 176 176 176 F F F 2 040-059 106 106 106 176 176 176 F F F 3 060-079 101 101 101 167 167 167 T T T 4 080-099 103 103 103 169 169 169 T T T 5 100-119 106 106 106 177 177 177 T T T 6 120-139 106 106 106 176 176 176 T T T 7 140-159 103 103 103 169 169 169 F F F 8 160-179 104 104 104 172 172 172 F F F 9 180-199 105 105 105 173 173 173 T T T 10 200-219 105 105 105 173 173 173 F F F 11 220-239 105 105 105 173 173 173 F F F 12 240-259 102 102 102 168 168 168 F F F 13 260-279 102 102 102 168 168 168 F F F 14 280-299 106 106 106 176 176 176 F F F 15 300-319 100 100 100 165 165 165 F F F 16 320-339 105 105 105 173 173 173 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 105 105 105 173 173 173 T T T 19 380-399 104 104 104 172 172 172 T T T 20 400-419 106 106 106 177 177 177 F F F 21 420-439 106 106 106 177 177 177 F F F 22 440-459 104 104 104 172 172 172 F F F 23 460-479 104 104 104 172 172 172 F F F 24 480-499 106 106 106 176 176 176 F F F 25 500-519 105 105 105 173 173 173 F F F 26 520-539 106 106 106 175 175 175 T T T 27 540-559 106 106 106 175 175 175 T T T 28 560-579 106 106 106 177 177 177 F F F 29 580-599 106 106 106 177 177 177 F F F Max gen: 106, Max flush: 177 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 173 173 173 F F F 1 020-039 106 106 106 176 176 176 F F F 2 040-059 106 106 106 176 176 176 F F F 3 060-079 107 107 107 178 178 178 F F F 4 080-099 107 107 107 178 178 178 F F F 5 100-119 107 107 107 178 178 178 F F F 6 120-139 107 107 107 178 178 178 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 104 104 104 172 172 172 F F F 9 180-199 107 107 107 178 178 178 F F F 10 200-219 105 105 105 173 173 173 F F F 11 220-239 105 105 105 173 173 173 F F F 12 240-259 102 102 102 168 168 168 F F F 13 260-279 102 102 102 168 168 168 F F F 14 280-299 106 106 106 176 176 176 F F F 15 300-319 100 100 100 165 165 165 F F F 16 320-339 105 105 105 173 173 173 F F F 17 340-359 104 104 104 170 170 170 F F F 18 360-379 107 107 107 178 178 178 F F F 19 380-399 107 107 107 178 178 178 F F F 20 400-419 106 106 106 177 177 177 F F F 21 420-439 106 106 106 177 177 177 F F F 22 440-459 104 104 104 172 172 172 F F F 23 460-479 104 104 104 172 172 172 F F F 24 480-499 106 106 106 176 176 176 F F F 25 500-519 105 105 105 173 173 173 F F F 26 520-539 107 107 107 178 178 178 F F F 27 540-559 107 107 107 178 178 178 F F F 28 560-579 106 106 106 177 177 177 F F F 29 580-599 106 106 106 177 177 177 F F F Max gen: 107, Max flush: 178 Begin loop 54 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:38.903880102Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:38.947217085Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:38.947669053Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:38.97880447Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:38.979619935Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:38.97965858Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:38.979673608Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:38.979687787Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619} {"msg":"Crucible 7405d319-3f9e-4cdd-a8a6-815406e49b7a has session id: 75e87565-aa9f-451f-b538-3b47f8454e94","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:38.980214815Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Upstairs opts: Upstairs UUID: 7405d319-3f9e-4cdd-a8a6-815406e49b7a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:38.980252771Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Crucible stats registered with UUID: 7405d319-3f9e-4cdd-a8a6-815406e49b7a","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:38.980269449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.01145666Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011515523Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619} {"msg":"7405d319-3f9e-4cdd-a8a6-815406e49b7a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011532351Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011548699Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"0","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011578275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"0","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011597153Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"1","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011615561Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"1","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011638718Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"2","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011657826Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"2","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011674504Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011690852Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011704741Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"ds_connection connected from Ok(127.0.0.1:56805)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011844434Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"ds_connection connected from Ok(127.0.0.1:49904)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011886699Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"ds_connection connected from Ok(127.0.0.1:53547)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.011913616Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.012719512Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"2","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.012750059Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"1","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.013187178Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"0","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.013431539Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"1","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.013580462Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"2","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.013906614Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"0","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014076864Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.01419876Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"[0]R flush_numbers[0..12]: [173, 176, 176, 178, 178, 178, 178, 169, 172, 178, 173, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.01453956Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"[0]R generation[0..12]: [105, 106, 106, 107, 107, 107, 107, 103, 104, 107, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014560118Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014576325Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"[1]R flush_numbers[0..12]: [173, 176, 176, 178, 178, 178, 178, 169, 172, 178, 173, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014590624Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"[1]R generation[0..12]: [105, 106, 106, 107, 107, 107, 107, 103, 104, 107, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014604882Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014619311Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"[2]R flush_numbers[0..12]: [173, 176, 176, 178, 178, 178, 178, 169, 172, 178, 173, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014633449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"[2]R generation[0..12]: [105, 106, 106, 107, 107, 107, 107, 103, 104, 107, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014647317Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014661456Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014674574Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014688063Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Next flush: 179","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2025-01-13T15:55:39.01470787Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014725418Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"":"downstairs","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014739737Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014751975Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"0","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014765783Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"1","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014779232Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"client":"2","":"downstairs","session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.01479275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014805339Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"7405d319-3f9e-4cdd-a8a6-815406e49b7a is now active with session: 75e87565-aa9f-451f-b538-3b47f8454e94","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014818357Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014830546Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619,"session_id":"75e87565-aa9f-451f-b538-3b47f8454e94"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:39.014843225Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 558 len 36864 data: 7 8 7 8 9 8 8 10 11 02/30 Read block 87 len 20480 03/30 Write block 497 len 16384 data: 6 8 11 14 04/30 Read block 557 len 8192 05/30 Read block 324 len 28672 06/30 Read block 505 len 32768 07/30 Flush 08/30 Write block 116 len 40960 data: 12 12 11 12 11 10 11 13 12 11 09/30 Read block 507 len 36864 10/30 Read block 49 len 28672 11/30 Read block 10 len 8192 12/30 Read block 79 len 24576 13/30 Write block 296 len 28672 data: 5 7 8 9 8 8 11 14/30 Read block 374 len 32768 15/30 Read block 483 len 16384 16/30 Write block 397 len 40960 data: 11 9 10 9 9 10 11 9 9 9 17/30 Write block 270 len 16384 data: 8 7 6 4 18/30 Read block 231 len 28672 19/30 Write block 231 len 8192 data: 12 12 20/30 Read block 55 len 16384 21/30 Write block 402 len 32768 data: 11 12 10 10 10 7 9 10 22/30 Read block 475 len 28672 23/30 Write block 359 len 16384 data: 10 8 7 8 24/30 Flush 25/30 Write block 95 len 28672 data: 11 10 9 9 7 5 5 26/30 Read block 37 len 24576 27/30 Write block 210 len 24576 data: 9 12 13 13 12 12 28/30 Write block 375 len 36864 data: 5 4 5 5 7 8 10 11 10 29/30 Write block 189 len 36864 data: 11 9 11 8 10 11 12 11 11 30/30 Read block 329 len 12288 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 9 Sent Sent Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 4 4 6 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:39.037027376Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1619} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 173 173 173 F F F 1 020-039 106 106 106 176 176 176 F F F 2 040-059 106 106 106 176 176 176 F F F 3 060-079 107 107 107 178 178 178 F F F 4 080-099 107 107 107 178 178 178 T T T 5 100-119 108 108 108 180 180 180 T T T 6 120-139 108 108 108 180 180 180 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 104 104 104 172 172 172 F F F 9 180-199 107 107 107 178 178 178 T T T 10 200-219 105 105 105 173 173 173 T T T 11 220-239 108 108 108 180 180 180 F F F 12 240-259 102 102 102 168 168 168 F F F 13 260-279 108 108 108 180 180 180 F F F 14 280-299 108 108 108 180 180 180 F F F 15 300-319 108 108 108 180 180 180 F F F 16 320-339 105 105 105 173 173 173 F F F 17 340-359 108 108 108 180 180 180 F F F 18 360-379 108 108 108 180 180 180 T T T 19 380-399 108 108 108 180 180 180 T T T 20 400-419 108 108 108 180 180 180 F F F 21 420-439 106 106 106 177 177 177 F F F 22 440-459 104 104 104 172 172 172 F F F 23 460-479 104 104 104 172 172 172 F F F 24 480-499 108 108 108 179 179 179 F F F 25 500-519 108 108 108 179 179 179 F F F 26 520-539 107 107 107 178 178 178 F F F 27 540-559 108 108 108 179 179 179 F F F 28 560-579 108 108 108 179 179 179 F F F 29 580-599 106 106 106 177 177 177 F F F Max gen: 108, Max flush: 180 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 173 173 173 F F F 1 020-039 106 106 106 176 176 176 F F F 2 040-059 106 106 106 176 176 176 F F F 3 060-079 107 107 107 178 178 178 F F F 4 080-099 109 109 109 181 181 181 F F F 5 100-119 109 109 109 181 181 181 F F F 6 120-139 108 108 108 180 180 180 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 104 104 104 172 172 172 F F F 9 180-199 109 109 109 181 181 181 F F F 10 200-219 109 109 109 181 181 181 F F F 11 220-239 108 108 108 180 180 180 F F F 12 240-259 102 102 102 168 168 168 F F F 13 260-279 108 108 108 180 180 180 F F F 14 280-299 108 108 108 180 180 180 F F F 15 300-319 108 108 108 180 180 180 F F F 16 320-339 105 105 105 173 173 173 F F F 17 340-359 108 108 108 180 180 180 F F F 18 360-379 109 109 109 181 181 181 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 108 108 108 180 180 180 F F F 21 420-439 106 106 106 177 177 177 F F F 22 440-459 104 104 104 172 172 172 F F F 23 460-479 104 104 104 172 172 172 F F F 24 480-499 108 108 108 179 179 179 F F F 25 500-519 108 108 108 179 179 179 F F F 26 520-539 107 107 107 178 178 178 F F F 27 540-559 108 108 108 179 179 179 F F F 28 560-579 108 108 108 179 179 179 F F F 29 580-599 106 106 106 177 177 177 F F F Max gen: 109, Max flush: 181 Begin loop 55 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.34725685Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:43.391699684Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.39224764Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.423971238Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.424752057Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.424787163Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.424802101Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.424816329Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627} {"msg":"Crucible 3b41d524-8c06-4a7e-98b3-2d9a46d50788 has session id: dd8d0be5-3273-4536-b73b-9191f3f405ce","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.425332059Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Upstairs opts: Upstairs UUID: 3b41d524-8c06-4a7e-98b3-2d9a46d50788, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.425384303Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Crucible stats registered with UUID: 3b41d524-8c06-4a7e-98b3-2d9a46d50788","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.425405871Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.455512988Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.455560492Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627} {"msg":"3b41d524-8c06-4a7e-98b3-2d9a46d50788 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.455589139Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.455607347Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"0","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.455645992Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"0","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.455681038Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"1","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.455716564Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"1","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.455766338Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"2","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.455802384Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"2","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.45583701Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.455871696Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.455904322Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:33128)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.455942017Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:42866)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.45617947Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:58596)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.456221005Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.456790678Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"1","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.456835213Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"2","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.456868449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"0","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.457429634Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"1","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.45746348Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"2","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.457510645Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"0","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458056811Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458095196Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"[0]R flush_numbers[0..12]: [173, 176, 176, 178, 181, 181, 180, 169, 172, 181, 181, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458110664Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"[0]R generation[0..12]: [105, 106, 106, 107, 109, 109, 108, 103, 104, 109, 109, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458125423Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458139651Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"[1]R flush_numbers[0..12]: [173, 176, 176, 178, 181, 181, 180, 169, 172, 181, 181, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458153759Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"[1]R generation[0..12]: [105, 106, 106, 107, 109, 109, 108, 103, 104, 109, 109, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458167688Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458181796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"[2]R flush_numbers[0..12]: [173, 176, 176, 178, 181, 181, 180, 169, 172, 181, 181, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458196474Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"[2]R generation[0..12]: [105, 106, 106, 107, 109, 109, 108, 103, 104, 109, 109, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458214203Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458228791Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458242049Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458255458Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458268496Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458281464Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"":"downstairs","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"No downstairs reconciliation required"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458303222Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458326579Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"0","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458352636Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"1","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458378513Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"client":"2","":"downstairs","session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.45840442Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458428797Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"3b41d524-8c06-4a7e-98b3-2d9a46d50788 is now active with session: dd8d0be5-3273-4536-b73b-9191f3f405ce","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458454424Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458469173Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627,"session_id":"dd8d0be5-3273-4536-b73b-9191f3f405ce"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:43.458485221Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 215 len 32768 data: 13 12 10 12 10 10 10 9 02/30 Read block 317 len 36864 03/30 Read block 296 len 32768 04/30 Write block 435 len 36864 data: 7 8 11 10 8 6 5 5 8 05/30 Read block 586 len 36864 06/30 Read block 308 len 20480 07/30 Read block 213 len 24576 08/30 Write block 17 len 4096 data: 10 09/30 Write block 506 len 16384 data: 8 7 8 8 10/30 Read block 568 len 16384 11/30 Write block 314 len 20480 data: 6 5 5 4 4 12/30 Write block 563 len 8192 data: 9 9 13/30 Write block 423 len 32768 data: 7 7 6 6 5 5 7 6 14/30 Read block 457 len 40960 15/30 Read block 167 len 20480 16/30 Write block 590 len 8192 data: 11 13 17/30 Write block 340 len 32768 data: 5 6 9 9 10 10 11 9 18/30 Read block 199 len 4096 19/30 Write block 275 len 16384 data: 7 9 8 8 20/30 Write block 544 len 24576 data: 8 9 11 9 7 7 21/30 Read block 239 len 36864 22/30 Read block 434 len 12288 23/30 Write block 576 len 32768 data: 8 9 8 6 8 9 8 10 24/30 Read block 468 len 8192 25/30 Read block 173 len 12288 26/30 Read block 323 len 4096 27/30 Write block 482 len 32768 data: 6 4 5 7 8 8 8 8 28/30 Write block 147 len 16384 data: 8 11 11 10 29/30 Write block 419 len 12288 data: 8 8 8 30/30 Read block 109 len 16384 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 4 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 3 3 Done 30 30 27 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:43.477844362Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1627} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 173 173 173 T T T 1 020-039 106 106 106 176 176 176 F F F 2 040-059 106 106 106 176 176 176 F F F 3 060-079 107 107 107 178 178 178 F F F 4 080-099 109 109 109 181 181 181 F F F 5 100-119 109 109 109 181 181 181 F F F 6 120-139 108 108 108 180 180 180 F F F 7 140-159 103 103 103 169 169 169 T T T 8 160-179 104 104 104 172 172 172 F F F 9 180-199 109 109 109 181 181 181 F F F 10 200-219 109 109 109 181 181 181 T T T 11 220-239 108 108 108 180 180 180 T T T 12 240-259 102 102 102 168 168 168 F F F 13 260-279 108 108 108 180 180 180 T T T 14 280-299 108 108 108 180 180 180 F F F 15 300-319 108 108 108 180 180 180 T T T 16 320-339 105 105 105 173 173 173 F F F 17 340-359 108 108 108 180 180 180 T T T 18 360-379 109 109 109 181 181 181 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 108 108 108 180 180 180 T T T 21 420-439 106 106 106 177 177 177 T T T 22 440-459 104 104 104 172 172 172 T T T 23 460-479 104 104 104 172 172 172 F F F 24 480-499 108 108 108 179 179 179 T T T 25 500-519 108 108 108 179 179 179 T T T 26 520-539 107 107 107 178 178 178 F F F 27 540-559 108 108 108 179 179 179 T T T 28 560-579 108 108 108 179 179 179 T T T 29 580-599 106 106 106 177 177 177 T T T Max gen: 109, Max flush: 181 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 182 182 182 F F F 1 020-039 106 106 106 176 176 176 F F F 2 040-059 106 106 106 176 176 176 F F F 3 060-079 107 107 107 178 178 178 F F F 4 080-099 109 109 109 181 181 181 F F F 5 100-119 109 109 109 181 181 181 F F F 6 120-139 108 108 108 180 180 180 F F F 7 140-159 110 110 110 182 182 182 F F F 8 160-179 104 104 104 172 172 172 F F F 9 180-199 109 109 109 181 181 181 F F F 10 200-219 110 110 110 182 182 182 F F F 11 220-239 110 110 110 182 182 182 F F F 12 240-259 102 102 102 168 168 168 F F F 13 260-279 110 110 110 182 182 182 F F F 14 280-299 108 108 108 180 180 180 F F F 15 300-319 110 110 110 182 182 182 F F F 16 320-339 105 105 105 173 173 173 F F F 17 340-359 110 110 110 182 182 182 F F F 18 360-379 109 109 109 181 181 181 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 110 110 110 182 182 182 F F F 21 420-439 110 110 110 182 182 182 F F F 22 440-459 110 110 110 182 182 182 F F F 23 460-479 104 104 104 172 172 172 F F F 24 480-499 110 110 110 182 182 182 F F F 25 500-519 110 110 110 182 182 182 F F F 26 520-539 107 107 107 178 178 178 F F F 27 540-559 110 110 110 182 182 182 F F F 28 560-579 110 110 110 182 182 182 F F F 29 580-599 110 110 110 182 182 182 F F F Max gen: 110, Max flush: 182 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.843057381Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.887497375Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.888357045Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.88839655Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.888411899Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.888426347Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635} {"msg":"Crucible 9efa7511-c29c-4ffc-aa73-fcc9fb7c96c1 has session id: 440a1c96-b7a4-496a-ad9e-448ff9cb5e6b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.888885513Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Upstairs opts: Upstairs UUID: 9efa7511-c29c-4ffc-aa73-fcc9fb7c96c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.888932458Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Crucible stats registered with UUID: 9efa7511-c29c-4ffc-aa73-fcc9fb7c96c1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.888957145Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.920931444Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.920992107Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635} {"msg":"9efa7511-c29c-4ffc-aa73-fcc9fb7c96c1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.921009165Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.921025793Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"0","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.921055199Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"0","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.921072687Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"1","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.921090275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"1","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.921125501Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"2","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.921160717Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"2","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.921192943Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.92122511Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.921255546Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35295)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.921399919Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52944)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.921437795Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58811)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.921456643Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.922273967Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"1","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.922319522Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"0","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.922464285Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"2","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.922942689Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"1","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923036198Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"0","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923092532Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"2","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923485096Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923579785Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"[0]R flush_numbers[0..12]: [182, 176, 176, 178, 181, 181, 180, 182, 172, 181, 182, 182]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923712589Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"[0]R generation[0..12]: [110, 106, 106, 107, 109, 109, 108, 110, 104, 109, 110, 110]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923735827Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923751825Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"[1]R flush_numbers[0..12]: [182, 176, 176, 178, 181, 181, 180, 182, 172, 181, 182, 182]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923766143Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"[1]R generation[0..12]: [110, 106, 106, 107, 109, 109, 108, 110, 104, 109, 110, 110]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923780321Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.92379481Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"[2]R flush_numbers[0..12]: [182, 176, 176, 178, 181, 181, 180, 182, 172, 181, 182, 182]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923809518Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"[2]R generation[0..12]: [110, 106, 106, 107, 109, 109, 108, 110, 104, 109, 110, 110]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923823626Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923837535Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Max found gen is 111","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923850673Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Generation requested: 112 >= found:111","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923864032Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.92387698Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923893588Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923915486Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923927624Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"0","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923940973Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"1","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923954101Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"client":"2","":"downstairs","session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923967379Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923979528Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"9efa7511-c29c-4ffc-aa73-fcc9fb7c96c1 is now active with session: 440a1c96-b7a4-496a-ad9e-448ff9cb5e6b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.923995076Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.924017994Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635,"session_id":"440a1c96-b7a4-496a-ad9e-448ff9cb5e6b"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:47.924040771Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 466 len 12288 data: 8 9 8 02/30 Read block 101 len 12288 03/30 Read block 376 len 28672 04/30 Write block 158 len 24576 data: 9 9 12 14 15 13 05/30 Read block 363 len 36864 06/30 Write block 447 len 36864 data: 5 5 7 7 8 8 8 7 9 07/30 Write block 156 len 40960 data: 7 8 10 10 13 15 16 14 13 11 08/30 Write block 415 len 40960 data: 11 10 8 7 9 9 9 7 8 8 09/30 Write block 334 len 40960 data: 5 4 6 7 7 6 6 7 10 10 10/30 Read block 485 len 24576 11/30 Flush 12/30 Write block 103 len 28672 data: 7 9 11 8 9 9 7 13/30 Read block 128 len 12288 14/30 Write block 18 len 4096 data: 12 15/30 Read block 260 len 28672 16/30 Write block 223 len 24576 data: 8 9 7 8 8 11 17/30 Write block 449 len 12288 data: 8 8 9 18/30 Read block 565 len 36864 19/30 Write block 108 len 8192 data: 10 8 20/30 Read block 542 len 4096 21/30 Read block 104 len 36864 22/30 Write block 486 len 16384 data: 9 9 9 9 23/30 Flush 24/30 Flush 25/30 Write block 96 len 24576 data: 11 10 10 8 6 6 26/30 Write block 16 len 16384 data: 12 11 13 14 27/30 Read block 393 len 24576 28/30 Read block 514 len 32768 29/30 Write block 36 len 24576 data: 6 7 9 10 9 11 30/30 Write block 542 len 32768 data: 8 8 9 10 12 10 8 8 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:47.944487105Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1635} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 184 184 184 T T T 1 020-039 106 106 106 176 176 176 T T T 2 040-059 106 106 106 176 176 176 T T T 3 060-079 107 107 107 178 178 178 F F F 4 080-099 109 109 109 181 181 181 T T T 5 100-119 112 112 112 184 184 184 T T T 6 120-139 108 108 108 180 180 180 F F F 7 140-159 112 112 112 183 183 183 F F F 8 160-179 112 112 112 183 183 183 F F F 9 180-199 109 109 109 181 181 181 F F F 10 200-219 110 110 110 182 182 182 F F F 11 220-239 112 112 112 184 184 184 F F F 12 240-259 102 102 102 168 168 168 F F F 13 260-279 110 110 110 182 182 182 F F F 14 280-299 108 108 108 180 180 180 F F F 15 300-319 110 110 110 182 182 182 F F F 16 320-339 112 112 112 183 183 183 F F F 17 340-359 112 112 112 183 183 183 F F F 18 360-379 109 109 109 181 181 181 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 112 112 112 183 183 183 F F F 21 420-439 112 112 112 183 183 183 F F F 22 440-459 112 112 112 184 184 184 F F F 23 460-479 112 112 112 183 183 183 F F F 24 480-499 112 112 112 184 184 184 F F F 25 500-519 110 110 110 182 182 182 F F F 26 520-539 107 107 107 178 178 178 F F F 27 540-559 110 110 110 182 182 182 T T T 28 560-579 110 110 110 182 182 182 F F F 29 580-599 110 110 110 182 182 182 F F F Max gen: 112, Max flush: 184 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 185 185 185 F F F 1 020-039 113 113 113 185 185 185 F F F 2 040-059 113 113 113 185 185 185 F F F 3 060-079 107 107 107 178 178 178 F F F 4 080-099 113 113 113 185 185 185 F F F 5 100-119 113 113 113 185 185 185 F F F 6 120-139 108 108 108 180 180 180 F F F 7 140-159 112 112 112 183 183 183 F F F 8 160-179 112 112 112 183 183 183 F F F 9 180-199 109 109 109 181 181 181 F F F 10 200-219 110 110 110 182 182 182 F F F 11 220-239 112 112 112 184 184 184 F F F 12 240-259 102 102 102 168 168 168 F F F 13 260-279 110 110 110 182 182 182 F F F 14 280-299 108 108 108 180 180 180 F F F 15 300-319 110 110 110 182 182 182 F F F 16 320-339 112 112 112 183 183 183 F F F 17 340-359 112 112 112 183 183 183 F F F 18 360-379 109 109 109 181 181 181 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 112 112 112 183 183 183 F F F 21 420-439 112 112 112 183 183 183 F F F 22 440-459 112 112 112 184 184 184 F F F 23 460-479 112 112 112 183 183 183 F F F 24 480-499 112 112 112 184 184 184 F F F 25 500-519 110 110 110 182 182 182 F F F 26 520-539 107 107 107 178 178 178 F F F 27 540-559 113 113 113 185 185 185 F F F 28 560-579 110 110 110 182 182 182 F F F 29 580-599 110 110 110 182 182 182 F F F Max gen: 113, Max flush: 185 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.233334195Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:52.278259173Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.278857564Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.311312756Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.312204252Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.312240788Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.312255107Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.312274494Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643} {"msg":"Crucible d0fc121a-ffa1-4c03-b018-2d2a7c09d434 has session id: 09519e6c-68ff-4c74-a95c-06d76cb8a92d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.312853637Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Upstairs opts: Upstairs UUID: d0fc121a-ffa1-4c03-b018-2d2a7c09d434, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.312899871Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Crucible stats registered with UUID: d0fc121a-ffa1-4c03-b018-2d2a7c09d434","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.312928978Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343264568Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343323001Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643} {"msg":"d0fc121a-ffa1-4c03-b018-2d2a7c09d434 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343340549Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343357257Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"0","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343387234Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"0","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343404832Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"1","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.34342286Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"1","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343487142Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"2","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343529417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"2","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343551075Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343569673Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343600129Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51889)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343650963Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46860)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343689619Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"ds_connection connected from Ok(127.0.0.1:50025)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.343725964Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.344473247Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"2","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.344517552Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"1","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.344675204Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"0","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.345300441Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"1","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.345401029Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"2","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.345447173Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"0","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.345942036Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.345974702Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"[0]R flush_numbers[0..12]: [185, 185, 185, 178, 185, 185, 180, 183, 183, 181, 182, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.34598969Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"[0]R generation[0..12]: [113, 113, 113, 107, 113, 113, 108, 112, 112, 109, 110, 112]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346005079Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346019787Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"[1]R flush_numbers[0..12]: [185, 185, 185, 178, 185, 185, 180, 183, 183, 181, 182, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346033885Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"[1]R generation[0..12]: [113, 113, 113, 107, 113, 113, 108, 112, 112, 109, 110, 112]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346047723Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346061922Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"[2]R flush_numbers[0..12]: [185, 185, 185, 178, 185, 185, 180, 183, 183, 181, 182, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.34608083Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"[2]R generation[0..12]: [113, 113, 113, 107, 113, 113, 108, 112, 112, 109, 110, 112]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346095418Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346109556Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346122675Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346136043Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346149052Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.34616203Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"":"downstairs","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346175209Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346206045Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"0","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346241561Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"1","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346258259Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"client":"2","":"downstairs","session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346271857Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346285066Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"d0fc121a-ffa1-4c03-b018-2d2a7c09d434 is now active with session: 09519e6c-68ff-4c74-a95c-06d76cb8a92d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346297784Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346309743Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643,"session_id":"09519e6c-68ff-4c74-a95c-06d76cb8a92d"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:52.346322002Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 298 len 32768 data: 9 10 9 9 12 10 6 6 02/30 Read block 280 len 4096 03/30 Write block 107 len 24576 data: 10 11 9 7 6 7 04/30 Write block 55 len 4096 data: 12 05/30 Write block 575 len 8192 data: 10 9 06/30 Read block 223 len 16384 07/30 Read block 187 len 16384 08/30 Flush 09/30 Write block 181 len 20480 data: 11 10 10 10 9 10/30 Write block 592 len 28672 data: 12 12 12 9 8 6 5 11/30 Read block 400 len 16384 12/30 Flush 13/30 Write block 345 len 16384 data: 11 12 10 10 14/30 Read block 369 len 8192 15/30 Write block 417 len 40960 data: 9 8 10 10 10 8 9 9 7 7 16/30 Read block 575 len 8192 17/30 Write block 410 len 24576 data: 13 11 13 14 13 12 18/30 Write block 327 len 24576 data: 9 10 7 7 6 7 19/30 Read block 321 len 32768 20/30 Write block 105 len 16384 data: 12 9 11 12 21/30 Read block 2 len 32768 22/30 Read block 222 len 20480 23/30 Read block 344 len 20480 24/30 Read block 523 len 12288 25/30 Read block 195 len 20480 26/30 Flush 27/30 Write block 234 len 28672 data: 6 7 5 7 8 6 5 28/30 Read block 553 len 36864 29/30 Write block 70 len 12288 data: 10 9 7 30/30 Write block 333 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:52.366680366Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1643} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 185 185 185 F F F 1 020-039 113 113 113 185 185 185 F F F 2 040-059 114 114 114 186 186 186 F F F 3 060-079 107 107 107 178 178 178 T T T 4 080-099 113 113 113 185 185 185 F F F 5 100-119 114 114 114 188 188 188 F F F 6 120-139 108 108 108 180 180 180 F F F 7 140-159 112 112 112 183 183 183 F F F 8 160-179 112 112 112 183 183 183 F F F 9 180-199 114 114 114 187 187 187 F F F 10 200-219 110 110 110 182 182 182 F F F 11 220-239 112 112 112 184 184 184 T T T 12 240-259 102 102 102 168 168 168 T T T 13 260-279 110 110 110 182 182 182 F F F 14 280-299 114 114 114 186 186 186 F F F 15 300-319 114 114 114 186 186 186 F F F 16 320-339 114 114 114 188 188 188 T T T 17 340-359 114 114 114 188 188 188 F F F 18 360-379 109 109 109 181 181 181 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 114 114 114 188 188 188 F F F 21 420-439 114 114 114 188 188 188 F F F 22 440-459 112 112 112 184 184 184 F F F 23 460-479 112 112 112 183 183 183 F F F 24 480-499 112 112 112 184 184 184 F F F 25 500-519 110 110 110 182 182 182 F F F 26 520-539 107 107 107 178 178 178 F F F 27 540-559 113 113 113 185 185 185 F F F 28 560-579 114 114 114 186 186 186 F F F 29 580-599 114 114 114 187 187 187 F F F Max gen: 114, Max flush: 188 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 185 185 185 F F F 1 020-039 113 113 113 185 185 185 F F F 2 040-059 114 114 114 186 186 186 F F F 3 060-079 115 115 115 189 189 189 F F F 4 080-099 113 113 113 185 185 185 F F F 5 100-119 114 114 114 188 188 188 F F F 6 120-139 108 108 108 180 180 180 F F F 7 140-159 112 112 112 183 183 183 F F F 8 160-179 112 112 112 183 183 183 F F F 9 180-199 114 114 114 187 187 187 F F F 10 200-219 110 110 110 182 182 182 F F F 11 220-239 115 115 115 189 189 189 F F F 12 240-259 115 115 115 189 189 189 F F F 13 260-279 110 110 110 182 182 182 F F F 14 280-299 114 114 114 186 186 186 F F F 15 300-319 114 114 114 186 186 186 F F F 16 320-339 115 115 115 189 189 189 F F F 17 340-359 114 114 114 188 188 188 F F F 18 360-379 109 109 109 181 181 181 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 114 114 114 188 188 188 F F F 21 420-439 114 114 114 188 188 188 F F F 22 440-459 112 112 112 184 184 184 F F F 23 460-479 112 112 112 183 183 183 F F F 24 480-499 112 112 112 184 184 184 F F F 25 500-519 110 110 110 182 182 182 F F F 26 520-539 107 107 107 178 178 178 F F F 27 540-559 113 113 113 185 185 185 F F F 28 560-579 114 114 114 186 186 186 F F F 29 580-599 114 114 114 187 187 187 F F F Max gen: 115, Max flush: 189 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.677554546Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.722253431Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.723142097Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.723190111Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.723208639Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.723226837Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651} {"msg":"Crucible c303ebfb-794a-432c-bc98-56f2449fcfb9 has session id: 6880d0e4-e976-49d0-a355-094a32ecc7c2","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.723704371Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Upstairs opts: Upstairs UUID: c303ebfb-794a-432c-bc98-56f2449fcfb9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.723763214Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Crucible stats registered with UUID: c303ebfb-794a-432c-bc98-56f2449fcfb9","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.723793051Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.75413408Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754184085Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651} {"msg":"c303ebfb-794a-432c-bc98-56f2449fcfb9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754205712Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.75422359Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"0","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754252737Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"0","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754270505Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"1","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754288242Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"1","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754316829Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"2","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754337577Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"2","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754354215Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754371423Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754388561Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43213)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754663069Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:62364)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754862706Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43600)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.754918019Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.755519069Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"0","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.755561354Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"1","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.755579672Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"2","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.756207249Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"1","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.756249564Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"2","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.75628071Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"0","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.756855163Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.75688219Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"[0]R flush_numbers[0..12]: [185, 185, 186, 189, 185, 188, 180, 183, 183, 187, 182, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.756989897Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"[0]R generation[0..12]: [113, 113, 114, 115, 113, 114, 108, 112, 112, 114, 110, 115]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757011065Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757025543Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"[1]R flush_numbers[0..12]: [185, 185, 186, 189, 185, 188, 180, 183, 183, 187, 182, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757044321Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"[1]R generation[0..12]: [113, 113, 114, 115, 113, 114, 108, 112, 112, 114, 110, 115]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757058699Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757073797Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"[2]R flush_numbers[0..12]: [185, 185, 186, 189, 185, 188, 180, 183, 183, 187, 182, 189]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757087826Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"[2]R generation[0..12]: [113, 113, 114, 115, 113, 114, 108, 112, 112, 114, 110, 115]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757101674Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757115823Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757130011Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757143569Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757157518Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:55:56.757170806Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"":"downstairs","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757190014Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757202213Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"0","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757215931Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"1","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757230129Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"client":"2","":"downstairs","session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757243588Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757258796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"c303ebfb-794a-432c-bc98-56f2449fcfb9 is now active with session: 6880d0e4-e976-49d0-a355-094a32ecc7c2","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757271864Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757284063Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651,"session_id":"6880d0e4-e976-49d0-a355-094a32ecc7c2"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:55:56.757296991Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 568 len 32768 data: 10 12 11 12 16 13 12 11 04/30 Read block 82 len 20480 05/30 Read block 14 len 32768 06/30 Read block 475 len 16384 07/30 Flush 08/30 Write block 521 len 40960 data: 9 8 7 7 4 4 5 7 8 10 09/30 Write block 230 len 20480 data: 12 13 13 7 7 10/30 Write block 140 len 40960 data: 9 10 9 9 12 9 8 9 12 12 11/30 Write block 419 len 16384 data: 11 11 11 9 12/30 Write block 485 len 20480 data: 8 10 10 10 10 13/30 Write block 74 len 4096 data: 7 14/30 Read block 200 len 40960 15/30 Read block 248 len 24576 16/30 Write block 582 len 28672 data: 9 11 8 9 7 9 9 17/30 Flush 18/30 Write block 502 len 16384 data: 13 13 12 9 19/30 Read block 279 len 20480 20/30 Read block 508 len 36864 21/30 Read block 285 len 20480 22/30 Read block 397 len 40960 23/30 Read block 425 len 16384 24/30 Read block 162 len 20480 25/30 Flush 26/30 Write block 315 len 40960 data: 6 6 5 5 3 8 8 9 9 8 27/30 Write block 38 len 8192 data: 10 11 28/30 Write block 2 len 24576 data: 3 4 5 5 7 11 29/30 Read block 153 len 28672 30/30 Read block 467 len 28672 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 5 4 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:55:56.777111879Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1651} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 185 185 185 T T T 1 020-039 113 113 113 185 185 185 T T T 2 040-059 114 114 114 186 186 186 F F F 3 060-079 116 116 116 193 193 193 F F F 4 080-099 113 113 113 185 185 185 F F F 5 100-119 114 114 114 188 188 188 F F F 6 120-139 108 108 108 180 180 180 F F F 7 140-159 116 116 116 193 193 193 F F F 8 160-179 112 112 112 183 183 183 F F F 9 180-199 114 114 114 187 187 187 F F F 10 200-219 110 110 110 182 182 182 F F F 11 220-239 116 116 116 193 193 193 F F F 12 240-259 115 115 115 189 189 189 F F F 13 260-279 110 110 110 182 182 182 F F F 14 280-299 114 114 114 186 186 186 F F F 15 300-319 114 114 114 186 186 186 T T T 16 320-339 115 115 115 189 189 189 T T T 17 340-359 114 114 114 188 188 188 F F F 18 360-379 109 109 109 181 181 181 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 116 116 116 193 193 193 F F F 21 420-439 116 116 116 193 193 193 F F F 22 440-459 112 112 112 184 184 184 F F F 23 460-479 112 112 112 183 183 183 F F F 24 480-499 116 116 116 193 193 193 F F F 25 500-519 116 116 116 194 194 194 F F F 26 520-539 116 116 116 193 193 193 F F F 27 540-559 113 113 113 185 185 185 F F F 28 560-579 116 116 116 192 192 192 F F F 29 580-599 116 116 116 193 193 193 F F F Max gen: 116, Max flush: 194 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 117 117 117 195 195 195 F F F 1 020-039 117 117 117 195 195 195 F F F 2 040-059 114 114 114 186 186 186 F F F 3 060-079 116 116 116 193 193 193 F F F 4 080-099 113 113 113 185 185 185 F F F 5 100-119 114 114 114 188 188 188 F F F 6 120-139 108 108 108 180 180 180 F F F 7 140-159 116 116 116 193 193 193 F F F 8 160-179 112 112 112 183 183 183 F F F 9 180-199 114 114 114 187 187 187 F F F 10 200-219 110 110 110 182 182 182 F F F 11 220-239 116 116 116 193 193 193 F F F 12 240-259 115 115 115 189 189 189 F F F 13 260-279 110 110 110 182 182 182 F F F 14 280-299 114 114 114 186 186 186 F F F 15 300-319 117 117 117 195 195 195 F F F 16 320-339 117 117 117 195 195 195 F F F 17 340-359 114 114 114 188 188 188 F F F 18 360-379 109 109 109 181 181 181 F F F 19 380-399 109 109 109 181 181 181 F F F 20 400-419 116 116 116 193 193 193 F F F 21 420-439 116 116 116 193 193 193 F F F 22 440-459 112 112 112 184 184 184 F F F 23 460-479 112 112 112 183 183 183 F F F 24 480-499 116 116 116 193 193 193 F F F 25 500-519 116 116 116 194 194 194 F F F 26 520-539 116 116 116 193 193 193 F F F 27 540-559 113 113 113 185 185 185 F F F 28 560-579 116 116 116 192 192 192 F F F 29 580-599 116 116 116 193 193 193 F F F Max gen: 117, Max flush: 195 Begin loop 59 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.058502959Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.104395144Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.105195841Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.105235186Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.105261303Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.105276461Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659} {"msg":"Crucible c8e42695-fa77-4098-b642-3a4aa074588c has session id: f06cc405-8021-4409-8533-3f8adb9a5e53","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.10571484Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Upstairs opts: Upstairs UUID: c8e42695-fa77-4098-b642-3a4aa074588c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.105757515Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Crucible stats registered with UUID: c8e42695-fa77-4098-b642-3a4aa074588c","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.105774763Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.136442664Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.13648366Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659} {"msg":"c8e42695-fa77-4098-b642-3a4aa074588c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.13656831Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.136591937Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"0","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.136621943Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"0","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.136664529Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"1","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.136683826Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"1","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.136700404Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"2","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.136724111Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"2","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.136741489Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.136758907Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.136780735Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"ds_connection connected from Ok(127.0.0.1:45645)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.136929698Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"ds_connection connected from Ok(127.0.0.1:49784)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.136991231Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"ds_connection connected from Ok(127.0.0.1:54417)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.137060772Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.137804016Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"2","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.137832302Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"0","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.137984635Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"1","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.138508333Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"0","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.138544259Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"2","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.138700861Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"1","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139081567Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139197133Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"[0]R flush_numbers[0..12]: [195, 195, 186, 193, 185, 188, 180, 193, 183, 187, 182, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139351885Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"[0]R generation[0..12]: [117, 117, 114, 116, 113, 114, 108, 116, 112, 114, 110, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139373742Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139388961Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"[1]R flush_numbers[0..12]: [195, 195, 186, 193, 185, 188, 180, 193, 183, 187, 182, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139403229Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"[1]R generation[0..12]: [117, 117, 114, 116, 113, 114, 108, 116, 112, 114, 110, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139417117Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139431236Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"[2]R flush_numbers[0..12]: [195, 195, 186, 193, 185, 188, 180, 193, 183, 187, 182, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139445094Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"[2]R generation[0..12]: [117, 117, 114, 116, 113, 114, 108, 116, 112, 114, 110, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139459003Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139473101Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139486249Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139500028Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139513196Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139526305Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"":"downstairs","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139546512Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139561611Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"0","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139575699Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"1","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139589427Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"client":"2","":"downstairs","session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139603206Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139615764Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"c8e42695-fa77-4098-b642-3a4aa074588c is now active with session: f06cc405-8021-4409-8533-3f8adb9a5e53","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139628713Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.139640691Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659,"session_id":"f06cc405-8021-4409-8533-3f8adb9a5e53"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:01.13965308Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 262 len 24576 data: 11 11 11 11 13 12 02/30 Read block 215 len 8192 03/30 Read block 256 len 16384 04/30 Read block 444 len 8192 05/30 Read block 539 len 32768 06/30 Read block 201 len 24576 07/30 Write block 253 len 28672 data: 8 10 11 8 9 10 11 08/30 Read block 569 len 24576 09/30 Write block 269 len 4096 data: 8 10/30 Read block 396 len 32768 11/30 Write block 23 len 24576 data: 14 14 13 14 13 11 12/30 Write block 375 len 16384 data: 6 5 6 6 13/30 Write block 256 len 20480 data: 9 10 11 12 12 14/30 Read block 142 len 40960 15/30 Flush 16/30 Flush 17/30 Flush 18/30 Write block 52 len 28672 data: 11 9 12 13 9 9 8 19/30 Write block 162 len 36864 data: 17 15 14 12 12 14 15 13 14 20/30 Read block 330 len 32768 21/30 Flush 22/30 Write block 580 len 4096 data: 9 23/30 Read block 497 len 24576 24/30 Write block 341 len 16384 data: 8 11 11 11 25/30 Write block 375 len 12288 data: 7 6 7 26/30 Write block 10 len 32768 data: 9 9 10 10 9 8 13 12 27/30 Write block 117 len 16384 data: 13 12 13 12 28/30 Read block 240 len 4096 29/30 Read block 25 len 28672 30/30 Write block 398 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:01.159050317Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1659} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 117 117 117 195 195 195 T T T 1 020-039 118 118 118 196 196 196 F F F 2 040-059 118 118 118 199 199 199 F F F 3 060-079 116 116 116 193 193 193 F F F 4 080-099 113 113 113 185 185 185 F F F 5 100-119 114 114 114 188 188 188 T T T 6 120-139 108 108 108 180 180 180 T T T 7 140-159 116 116 116 193 193 193 F F F 8 160-179 118 118 118 199 199 199 F F F 9 180-199 114 114 114 187 187 187 F F F 10 200-219 110 110 110 182 182 182 F F F 11 220-239 116 116 116 193 193 193 F F F 12 240-259 118 118 118 196 196 196 F F F 13 260-279 118 118 118 196 196 196 F F F 14 280-299 114 114 114 186 186 186 F F F 15 300-319 117 117 117 195 195 195 F F F 16 320-339 117 117 117 195 195 195 F F F 17 340-359 114 114 114 188 188 188 T T T 18 360-379 118 118 118 196 196 196 T T T 19 380-399 109 109 109 181 181 181 T T T 20 400-419 116 116 116 193 193 193 F F F 21 420-439 116 116 116 193 193 193 F F F 22 440-459 112 112 112 184 184 184 F F F 23 460-479 112 112 112 183 183 183 F F F 24 480-499 116 116 116 193 193 193 F F F 25 500-519 116 116 116 194 194 194 F F F 26 520-539 116 116 116 193 193 193 F F F 27 540-559 113 113 113 185 185 185 F F F 28 560-579 116 116 116 192 192 192 F F F 29 580-599 116 116 116 193 193 193 T T T Max gen: 118, Max flush: 199 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 200 200 200 F F F 1 020-039 118 118 118 196 196 196 F F F 2 040-059 118 118 118 199 199 199 F F F 3 060-079 116 116 116 193 193 193 F F F 4 080-099 113 113 113 185 185 185 F F F 5 100-119 119 119 119 200 200 200 F F F 6 120-139 119 119 119 200 200 200 F F F 7 140-159 116 116 116 193 193 193 F F F 8 160-179 118 118 118 199 199 199 F F F 9 180-199 114 114 114 187 187 187 F F F 10 200-219 110 110 110 182 182 182 F F F 11 220-239 116 116 116 193 193 193 F F F 12 240-259 118 118 118 196 196 196 F F F 13 260-279 118 118 118 196 196 196 F F F 14 280-299 114 114 114 186 186 186 F F F 15 300-319 117 117 117 195 195 195 F F F 16 320-339 117 117 117 195 195 195 F F F 17 340-359 119 119 119 200 200 200 F F F 18 360-379 119 119 119 200 200 200 F F F 19 380-399 119 119 119 200 200 200 F F F 20 400-419 116 116 116 193 193 193 F F F 21 420-439 116 116 116 193 193 193 F F F 22 440-459 112 112 112 184 184 184 F F F 23 460-479 112 112 112 183 183 183 F F F 24 480-499 116 116 116 193 193 193 F F F 25 500-519 116 116 116 194 194 194 F F F 26 520-539 116 116 116 193 193 193 F F F 27 540-559 113 113 113 185 185 185 F F F 28 560-579 116 116 116 192 192 192 F F F 29 580-599 119 119 119 200 200 200 F F F Max gen: 119, Max flush: 200 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.445729228Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.489903954Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.490812538Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.490857733Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.490872561Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.490887659Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667} {"msg":"Crucible ddd30061-6112-404a-abcb-f8008b16cfa0 has session id: f4e8adc2-fd47-4e6e-91c7-e3c252461f98","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.49130912Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Upstairs opts: Upstairs UUID: ddd30061-6112-404a-abcb-f8008b16cfa0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.491360404Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Crucible stats registered with UUID: ddd30061-6112-404a-abcb-f8008b16cfa0","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.491389211Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.522566693Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.522647123Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667} {"msg":"ddd30061-6112-404a-abcb-f8008b16cfa0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.52267792Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.522708226Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"0","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.522767199Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"0","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.522809794Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"1","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.522874127Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"1","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.522908483Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"2","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.5229343Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"2","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.522955977Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.522978515Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.523001702Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"ds_connection connected from Ok(127.0.0.1:50462)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.523095791Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"ds_connection connected from Ok(127.0.0.1:46674)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.523152324Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"ds_connection connected from Ok(127.0.0.1:45737)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.52319243Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.524100204Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"2","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.52413066Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"1","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.524146378Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"0","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.5248133Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"1","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.524848127Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"2","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.524864645Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"0","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525444727Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525481853Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"[0]R flush_numbers[0..12]: [200, 196, 199, 193, 185, 200, 200, 193, 199, 187, 182, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.52550639Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"[0]R generation[0..12]: [119, 118, 118, 116, 113, 119, 119, 116, 118, 114, 110, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525522788Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525537436Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"[1]R flush_numbers[0..12]: [200, 196, 199, 193, 185, 200, 200, 193, 199, 187, 182, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525551394Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"[1]R generation[0..12]: [119, 118, 118, 116, 113, 119, 119, 116, 118, 114, 110, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525565453Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525579991Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"[2]R flush_numbers[0..12]: [200, 196, 199, 193, 185, 200, 200, 193, 199, 187, 182, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.52559402Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"[2]R generation[0..12]: [119, 118, 118, 116, 113, 119, 119, 116, 118, 114, 110, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525607868Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525622646Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525640524Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525654802Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525668061Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525690868Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"":"downstairs","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525722345Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525744762Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"0","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.52575957Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"1","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525772868Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"client":"2","":"downstairs","session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525786147Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525798345Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"ddd30061-6112-404a-abcb-f8008b16cfa0 is now active with session: f4e8adc2-fd47-4e6e-91c7-e3c252461f98","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525811474Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "2025-01-13T15:56:05.525823793Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667,"session_id":"f4e8adc2-fd47-4e6e-91c7-e3c252461f98"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:05.525839721Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 193 len 12288 data: 11 12 13 02/30 Read block 579 len 40960 03/30 Read block 507 len 28672 04/30 Read block 111 len 16384 05/30 Write block 351 len 8192 data: 7 6 06/30 Write block 23 len 24576 data: 15 15 14 15 14 12 07/30 Read block 227 len 28672 08/30 Read block 130 len 8192 09/30 Write block 492 len 28672 data: 7 5 7 6 7 7 9 10/30 Write block 411 len 4096 data: 12 11/30 Write block 45 len 20480 data: 10 10 8 9 9 12/30 Write block 443 len 36864 data: 9 8 6 7 6 6 9 9 10 13/30 Read block 234 len 4096 14/30 Write block 67 len 16384 data: 11 11 10 11 15/30 Read block 507 len 16384 16/30 Read block 261 len 4096 17/30 Read block 277 len 16384 18/30 Write block 497 len 28672 data: 8 10 12 15 14 14 14 19/30 Write block 361 len 24576 data: 8 9 8 8 8 8 20/30 Write block 131 len 40960 data: 5 4 5 7 8 8 8 7 7 10 21/30 Flush 22/30 Write block 46 len 8192 data: 11 9 23/30 Flush 24/30 Write block 228 len 28672 data: 12 10 13 14 14 8 8 25/30 Read block 416 len 12288 26/30 Read block 84 len 28672 27/30 Read block 334 len 36864 28/30 Write block 544 len 40960 data: 10 11 13 11 9 9 8 11 9 9 29/30 Write block 562 len 24576 data: 10 10 10 11 12 10 30/30 Write block 206 len 28672 data: 11 11 10 10 10 13 14 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:05.545484859Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1667} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 200 200 200 F F F 1 020-039 120 120 120 201 201 201 F F F 2 040-059 120 120 120 202 202 202 F F F 3 060-079 120 120 120 201 201 201 F F F 4 080-099 113 113 113 185 185 185 F F F 5 100-119 119 119 119 200 200 200 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 201 201 201 F F F 8 160-179 118 118 118 199 199 199 F F F 9 180-199 120 120 120 201 201 201 F F F 10 200-219 110 110 110 182 182 182 T T T 11 220-239 116 116 116 193 193 193 T T T 12 240-259 118 118 118 196 196 196 F F F 13 260-279 118 118 118 196 196 196 F F F 14 280-299 114 114 114 186 186 186 F F F 15 300-319 117 117 117 195 195 195 F F F 16 320-339 117 117 117 195 195 195 F F F 17 340-359 120 120 120 201 201 201 F F F 18 360-379 120 120 120 201 201 201 F F F 19 380-399 119 119 119 200 200 200 F F F 20 400-419 120 120 120 201 201 201 F F F 21 420-439 116 116 116 193 193 193 F F F 22 440-459 120 120 120 201 201 201 F F F 23 460-479 112 112 112 183 183 183 F F F 24 480-499 120 120 120 201 201 201 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 116 116 116 193 193 193 F F F 27 540-559 113 113 113 185 185 185 T T T 28 560-579 116 116 116 192 192 192 T T T 29 580-599 119 119 119 200 200 200 F F F Max gen: 120, Max flush: 202 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 200 200 200 F F F 1 020-039 120 120 120 201 201 201 F F F 2 040-059 120 120 120 202 202 202 F F F 3 060-079 120 120 120 201 201 201 F F F 4 080-099 113 113 113 185 185 185 F F F 5 100-119 119 119 119 200 200 200 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 201 201 201 F F F 8 160-179 118 118 118 199 199 199 F F F 9 180-199 120 120 120 201 201 201 F F F 10 200-219 121 121 121 203 203 203 F F F 11 220-239 121 121 121 203 203 203 F F F 12 240-259 118 118 118 196 196 196 F F F 13 260-279 118 118 118 196 196 196 F F F 14 280-299 114 114 114 186 186 186 F F F 15 300-319 117 117 117 195 195 195 F F F 16 320-339 117 117 117 195 195 195 F F F 17 340-359 120 120 120 201 201 201 F F F 18 360-379 120 120 120 201 201 201 F F F 19 380-399 119 119 119 200 200 200 F F F 20 400-419 120 120 120 201 201 201 F F F 21 420-439 116 116 116 193 193 193 F F F 22 440-459 120 120 120 201 201 201 F F F 23 460-479 112 112 112 183 183 183 F F F 24 480-499 120 120 120 201 201 201 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 116 116 116 193 193 193 F F F 27 540-559 121 121 121 203 203 203 F F F 28 560-579 121 121 121 203 203 203 F F F 29 580-599 119 119 119 200 200 200 F F F Max gen: 121, Max flush: 203 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.818492216Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.862434388Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.863054066Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.863117109Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.863136776Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.863154994Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675} {"msg":"Crucible 4538a38f-a4fe-4c79-b848-2b2959d8d01d has session id: 06d42546-7206-4d5c-af47-cb9c2b5ef6cd","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.863592053Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"Upstairs opts: Upstairs UUID: 4538a38f-a4fe-4c79-b848-2b2959d8d01d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.863639588Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"Crucible stats registered with UUID: 4538a38f-a4fe-4c79-b848-2b2959d8d01d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.863670234Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893376038Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893415723Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675} {"msg":"4538a38f-a4fe-4c79-b848-2b2959d8d01d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893476796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893504833Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"0","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.89352966Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"0","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893545088Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"1","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893559846Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"1","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893573435Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"2","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893587733Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"2","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893601122Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.8936151Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893628869Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:58834)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893790399Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:54159)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893844503Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:38384)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.893889168Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.894480069Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"2","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.894510335Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"0","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.894532333Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"1","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.895131233Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"0","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.895378174Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"1","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.895514358Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"2","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.895764969Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.895988143Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"[0]R flush_numbers[0..12]: [200, 201, 202, 201, 185, 200, 201, 201, 199, 201, 203, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896080362Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"[0]R generation[0..12]: [119, 120, 120, 120, 113, 119, 120, 120, 118, 120, 121, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896104909Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896120258Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"[1]R flush_numbers[0..12]: [200, 201, 202, 201, 185, 200, 201, 201, 199, 201, 203, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896134556Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"[1]R generation[0..12]: [119, 120, 120, 120, 113, 119, 120, 120, 118, 120, 121, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896148444Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896165682Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"[2]R flush_numbers[0..12]: [200, 201, 202, 201, 185, 200, 201, 201, 199, 201, 203, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896179811Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"[2]R generation[0..12]: [119, 120, 120, 120, 113, 119, 120, 120, 118, 120, 121, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896193769Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896207717Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896224275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896249093Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896278669Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896303346Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"":"downstairs","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896321614Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896338612Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"0","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896362979Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"1","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896386987Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"client":"2","":"downstairs","session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896404325Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896426752Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"session_idDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"4538a38f-a4fe-4c79-b848-2b2959d8d01d is now active with session: 06d42546-7206-4d5c-af47-cb9c2b5ef6cd","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896454599Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896480436Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675,"session_id":"06d42546-7206-4d5c-af47-cb9c2b5ef6cd"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:09.896497814Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 172 len 20480 data: 9 6 6 5 6 02/30 Write block 480 len 20480 data: 11 9 7 5 6 03/30 Read block 550 len 16384 04/30 Read block 73 len 40960 05/30 Flush 06/30 Write block 15 len 28672 data: 9 14 13 14 15 12 13 07/30 Read block 377 len 32768 08/30 Read block 378 len 40960 09/30 Read block 466 len 40960 10/30 Read block 120 len 32768 11/30 Write block 336 len 32768 data: 7 8 8 7 7 9 12 12 12/30 Write block 521 len 12288 data: 10 9 8 13/30 Write block 300 len 24576 data: 10 10 13 11 7 7 14/30 Read block 444 len 20480 15/30 Read block 254 len 24576 16/30 Read block 523 len 28672 17/30 Write block 594 len 24576 data: 13 10 9 7 6 5 18/30 Write block 587 len 4096 data: 10 19/30 Read block 63 len 20480 20/30 Read block 530 len 36864 21/30 Write block 476 len 4096 data: 8 22/30 Write block 430 len 36864 data: 7 6 9 8 7 8 9 12 11 23/30 Flush 24/30 Write block 357 len 16384 data: 7 8 11 9 25/30 Read block 354 len 28672 26/30 Read block 188 len 20480 27/30 Write block 325 len 24576 data: 11 9 10 11 8 8 28/30 Write block 473 len 12288 data: 5 7 7 29/30 Read block 203 len 40960 30/30 Write block 209 len 28672 data: 11 11 14 15 14 13 14 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:09.91658106Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1675} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 205 205 205 F F F 1 020-039 122 122 122 205 205 205 F F F 2 040-059 120 120 120 202 202 202 F F F 3 060-079 120 120 120 201 201 201 F F F 4 080-099 113 113 113 185 185 185 F F F 5 100-119 119 119 119 200 200 200 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 201 201 201 F F F 8 160-179 122 122 122 204 204 204 F F F 9 180-199 120 120 120 201 201 201 F F F 10 200-219 121 121 121 203 203 203 T T T 11 220-239 121 121 121 203 203 203 F F F 12 240-259 118 118 118 196 196 196 F F F 13 260-279 118 118 118 196 196 196 F F F 14 280-299 114 114 114 186 186 186 F F F 15 300-319 122 122 122 205 205 205 F F F 16 320-339 122 122 122 205 205 205 T T T 17 340-359 122 122 122 205 205 205 T T T 18 360-379 120 120 120 201 201 201 T T T 19 380-399 119 119 119 200 200 200 F F F 20 400-419 120 120 120 201 201 201 F F F 21 420-439 122 122 122 205 205 205 F F F 22 440-459 120 120 120 201 201 201 F F F 23 460-479 122 122 122 205 205 205 T T T 24 480-499 122 122 122 204 204 204 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 122 122 122 205 205 205 F F F 27 540-559 121 121 121 203 203 203 F F F 28 560-579 121 121 121 203 203 203 F F F 29 580-599 122 122 122 205 205 205 F F F Max gen: 122, Max flush: 205 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 205 205 205 F F F 1 020-039 122 122 122 205 205 205 F F F 2 040-059 120 120 120 202 202 202 F F F 3 060-079 120 120 120 201 201 201 F F F 4 080-099 113 113 113 185 185 185 F F F 5 100-119 119 119 119 200 200 200 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 201 201 201 F F F 8 160-179 122 122 122 204 204 204 F F F 9 180-199 120 120 120 201 201 201 F F F 10 200-219 123 123 123 206 206 206 F F F 11 220-239 121 121 121 203 203 203 F F F 12 240-259 118 118 118 196 196 196 F F F 13 260-279 118 118 118 196 196 196 F F F 14 280-299 114 114 114 186 186 186 F F F 15 300-319 122 122 122 205 205 205 F F F 16 320-339 123 123 123 206 206 206 F F F 17 340-359 123 123 123 206 206 206 F F F 18 360-379 123 123 123 206 206 206 F F F 19 380-399 119 119 119 200 200 200 F F F 20 400-419 120 120 120 201 201 201 F F F 21 420-439 122 122 122 205 205 205 F F F 22 440-459 120 120 120 201 201 201 F F F 23 460-479 123 123 123 206 206 206 F F F 24 480-499 122 122 122 204 204 204 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 122 122 122 205 205 205 F F F 27 540-559 121 121 121 203 203 203 F F F 28 560-579 121 121 121 203 203 203 F F F 29 580-599 122 122 122 205 205 205 F F F Max gen: 123, Max flush: 206 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.192445585Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:14.235181808Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.235605508Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.266575705Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.267171475Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.267207381Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.267225529Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.267239687Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683} {"msg":"Crucible a50eab8d-4773-4538-8faa-a8447d8094e6 has session id: 5538dee8-3495-4e8e-ada2-161682da9a67","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.267716072Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"Upstairs opts: Upstairs UUID: a50eab8d-4773-4538-8faa-a8447d8094e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.267755327Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"Crucible stats registered with UUID: a50eab8d-4773-4538-8faa-a8447d8094e6","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.267772385Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.297578897Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.297617863Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683} {"msg":"a50eab8d-4773-4538-8faa-a8447d8094e6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.297636421Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.297652468Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"0","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.297679785Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"0","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.297697263Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"1","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.297715121Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"1","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.297731729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"2","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.297755247Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"2","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.297772205Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.297789483Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.29781526Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"ds_connection connected from Ok(127.0.0.1:32994)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.297956023Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"ds_connection connected from Ok(127.0.0.1:59873)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.298009387Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"ds_connection connected from Ok(127.0.0.1:61832)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.298046742Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.298792786Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"1","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.29883601Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"2","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.298956916Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"0","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.29952904Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"1","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.299570375Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"2","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.299599791Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"0","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300141498Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300177344Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"[0]R flush_numbers[0..12]: [205, 205, 202, 201, 185, 200, 201, 201, 204, 201, 206, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300239607Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"[0]R generation[0..12]: [122, 122, 120, 120, 113, 119, 120, 120, 122, 120, 123, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300269883Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300286011Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"[1]R flush_numbers[0..12]: [205, 205, 202, 201, 185, 200, 201, 201, 204, 201, 206, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300300599Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"[1]R generation[0..12]: [122, 122, 120, 120, 113, 119, 120, 120, 122, 120, 123, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300314748Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300329236Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"[2]R flush_numbers[0..12]: [205, 205, 202, 201, 185, 200, 201, 201, 204, 201, 206, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300343314Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"[2]R generation[0..12]: [122, 122, 120, 120, 113, 119, 120, 120, 122, 120, 123, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300357203Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300371311Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.3003845Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300397848Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300418276Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300431464Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"":"downstairs","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300445133Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300460841Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"0","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300474869Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"1","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300488398Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"client":"2","":"downstairs","session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300501916Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300514405Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"a50eab8d-4773-4538-8faa-a8447d8094e6 is now active with session: 5538dee8-3495-4e8e-ada2-161682da9a67","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300527313Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.300540582Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683,"session_id":"5538dee8-3495-4e8e-ada2-161682da9a67"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:14.30055296Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 31 len 4096 data: 8 02/30 Write block 452 len 20480 data: 9 9 8 10 9 03/30 Read block 397 len 40960 04/30 Read block 287 len 4096 05/30 Read block 558 len 16384 06/30 Flush 07/30 Write block 298 len 40960 data: 10 11 11 11 14 12 8 8 6 6 08/30 Write block 101 len 16384 data: 7 7 8 10 09/30 Read block 337 len 4096 10/30 Read block 176 len 24576 11/30 Write block 402 len 32768 data: 12 13 11 11 11 8 10 11 12/30 Flush 13/30 Write block 190 len 12288 data: 10 12 9 14/30 Read block 424 len 12288 15/30 Read block 77 len 20480 16/30 Read block 570 len 24576 17/30 Read block 305 len 28672 18/30 Read block 414 len 12288 19/30 Write block 318 len 36864 data: 6 4 9 9 10 10 9 12 10 20/30 Write block 49 len 24576 data: 10 10 9 12 10 13 21/30 Read block 375 len 28672 22/30 Read block 394 len 28672 23/30 Write block 398 len 36864 data: 11 11 10 10 13 14 12 12 12 24/30 Read block 338 len 32768 25/30 Read block 567 len 16384 26/30 Flush 27/30 Write block 261 len 32768 data: 12 12 12 12 12 14 13 9 28/30 Write block 93 len 4096 data: 15 29/30 Read block 300 len 24576 30/30 Read block 126 len 16384 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:14.321328506Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1683} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 205 205 205 F F F 1 020-039 124 124 124 207 207 207 F F F 2 040-059 124 124 124 209 209 209 F F F 3 060-079 120 120 120 201 201 201 F F F 4 080-099 113 113 113 185 185 185 T T T 5 100-119 124 124 124 208 208 208 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 201 201 201 F F F 8 160-179 122 122 122 204 204 204 F F F 9 180-199 124 124 124 209 209 209 F F F 10 200-219 123 123 123 206 206 206 F F F 11 220-239 121 121 121 203 203 203 F F F 12 240-259 118 118 118 196 196 196 F F F 13 260-279 118 118 118 196 196 196 T T T 14 280-299 124 124 124 208 208 208 F F F 15 300-319 124 124 124 209 209 209 F F F 16 320-339 124 124 124 209 209 209 F F F 17 340-359 123 123 123 206 206 206 F F F 18 360-379 123 123 123 206 206 206 F F F 19 380-399 124 124 124 209 209 209 F F F 20 400-419 124 124 124 209 209 209 F F F 21 420-439 122 122 122 205 205 205 F F F 22 440-459 124 124 124 207 207 207 F F F 23 460-479 123 123 123 206 206 206 F F F 24 480-499 122 122 122 204 204 204 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 122 122 122 205 205 205 F F F 27 540-559 121 121 121 203 203 203 F F F 28 560-579 121 121 121 203 203 203 F F F 29 580-599 122 122 122 205 205 205 F F F Max gen: 124, Max flush: 209 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 205 205 205 F F F 1 020-039 124 124 124 207 207 207 F F F 2 040-059 124 124 124 209 209 209 F F F 3 060-079 120 120 120 201 201 201 F F F 4 080-099 125 125 125 210 210 210 F F F 5 100-119 124 124 124 208 208 208 F F F 6 120-139 120 120 120 201 201 201 F F F 7 140-159 120 120 120 201 201 201 F F F 8 160-179 122 122 122 204 204 204 F F F 9 180-199 124 124 124 209 209 209 F F F 10 200-219 123 123 123 206 206 206 F F F 11 220-239 121 121 121 203 203 203 F F F 12 240-259 118 118 118 196 196 196 F F F 13 260-279 125 125 125 210 210 210 F F F 14 280-299 124 124 124 208 208 208 F F F 15 300-319 124 124 124 209 209 209 F F F 16 320-339 124 124 124 209 209 209 F F F 17 340-359 123 123 123 206 206 206 F F F 18 360-379 123 123 123 206 206 206 F F F 19 380-399 124 124 124 209 209 209 F F F 20 400-419 124 124 124 209 209 209 F F F 21 420-439 122 122 122 205 205 205 F F F 22 440-459 124 124 124 207 207 207 F F F 23 460-479 123 123 123 206 206 206 F F F 24 480-499 122 122 122 204 204 204 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 122 122 122 205 205 205 F F F 27 540-559 121 121 121 203 203 203 F F F 28 560-579 121 121 121 203 203 203 F F F 29 580-599 122 122 122 205 205 205 F F F Max gen: 125, Max flush: 210 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.620634016Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.665651263Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.666526531Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.666563737Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.666579025Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.666593503Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691} {"msg":"Crucible ad8094a8-9f18-4b1f-9f63-5a39e3159fc1 has session id: 8c532883-07a9-4647-83f0-23c9aa844815","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.667149639Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Upstairs opts: Upstairs UUID: ad8094a8-9f18-4b1f-9f63-5a39e3159fc1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.667191863Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Crucible stats registered with UUID: ad8094a8-9f18-4b1f-9f63-5a39e3159fc1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.667209201Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.697836718Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.697912749Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691} {"msg":"ad8094a8-9f18-4b1f-9f63-5a39e3159fc1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.697941775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.697972902Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"0","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.698022136Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"0","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.698067331Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"1","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.698203895Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"1","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.69824514Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"2","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.698267337Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"2","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.698284395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.698305193Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.698338199Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"ds_connection connected from Ok(127.0.0.1:53657)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.698544675Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"ds_connection connected from Ok(127.0.0.1:55874)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.698561163Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"ds_connection connected from Ok(127.0.0.1:49943)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.698580751Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.699180161Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"0","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.699372448Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"1","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.69944549Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"2","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.70004723Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"1","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700083016Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"0","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700124931Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"2","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700651779Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700688205Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"[0]R flush_numbers[0..12]: [205, 207, 209, 201, 210, 208, 201, 201, 204, 209, 206, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700793633Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"[0]R generation[0..12]: [122, 124, 124, 120, 125, 124, 120, 120, 122, 124, 123, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.70081463Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700829338Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"[1]R flush_numbers[0..12]: [205, 207, 209, 201, 210, 208, 201, 201, 204, 209, 206, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700843067Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"[1]R generation[0..12]: [122, 124, 124, 120, 125, 124, 120, 120, 122, 124, 123, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700857035Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700870934Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"[2]R flush_numbers[0..12]: [205, 207, 209, 201, 210, 208, 201, 201, 204, 209, 206, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700884832Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"[2]R generation[0..12]: [122, 124, 124, 120, 125, 124, 120, 120, 122, 124, 123, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.70089879Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700912749Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700925977Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700943445Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700957083Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700970022Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"":"downstairs","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.70098334Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.700995559Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"0","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-01-13T15:56:18.701016507Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"1","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.701029885Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"client":"2","":"downstairs","session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.701043173Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.701055542Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"ad8094a8-9f18-4b1f-9f63-5a39e3159fc1 is now active with session: 8c532883-07a9-4647-83f0-23c9aa844815","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.701068261Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.701080199Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691,"session_id":"8c532883-07a9-4647-83f0-23c9aa844815"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:18.701092568Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 493 len 28672 data: 6 8 7 8 9 11 13 02/30 Read block 11 len 36864 03/30 Write block 520 len 4096 data: 10 04/30 Write block 292 len 24576 data: 13 11 11 7 6 8 05/30 Write block 135 len 16384 data: 9 9 9 8 06/30 Flush 07/30 Write block 301 len 32768 data: 12 15 13 9 9 7 7 6 08/30 Write block 257 len 28672 data: 11 12 13 13 13 13 13 09/30 Write block 251 len 12288 data: 6 7 9 10/30 Read block 310 len 4096 11/30 Write block 64 len 20480 data: 6 6 9 12 12 12/30 Write block 188 len 16384 data: 9 12 11 13 13/30 Flush 14/30 Write block 593 len 16384 data: 13 14 11 10 15/30 Write block 338 len 16384 data: 9 8 8 10 16/30 Read block 236 len 28672 17/30 Read block 14 len 36864 18/30 Read block 366 len 12288 19/30 Flush 20/30 Write block 564 len 28672 data: 11 12 13 11 11 13 12 21/30 Read block 436 len 28672 22/30 Flush 23/30 Write block 403 len 8192 data: 15 13 24/30 Write block 425 len 4096 data: 8 25/30 Flush 26/30 Write block 132 len 20480 data: 5 6 8 10 10 27/30 Read block 571 len 36864 28/30 Write block 536 len 36864 data: 11 14 13 13 11 10 9 9 11 29/30 Write block 217 len 32768 data: 11 13 11 11 11 10 9 10 30/30 Read block 541 len 36864 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:18.719467694Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1691} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 205 205 205 F F F 1 020-039 124 124 124 207 207 207 F F F 2 040-059 124 124 124 209 209 209 F F F 3 060-079 126 126 126 212 212 212 F F F 4 080-099 125 125 125 210 210 210 F F F 5 100-119 124 124 124 208 208 208 F F F 6 120-139 126 126 126 211 211 211 T T T 7 140-159 120 120 120 201 201 201 F F F 8 160-179 122 122 122 204 204 204 F F F 9 180-199 126 126 126 212 212 212 F F F 10 200-219 123 123 123 206 206 206 T T T 11 220-239 121 121 121 203 203 203 T T T 12 240-259 126 126 126 212 212 212 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 126 126 126 211 211 211 F F F 15 300-319 126 126 126 212 212 212 F F F 16 320-339 126 126 126 213 213 213 F F F 17 340-359 126 126 126 213 213 213 F F F 18 360-379 123 123 123 206 206 206 F F F 19 380-399 124 124 124 209 209 209 F F F 20 400-419 126 126 126 215 215 215 F F F 21 420-439 126 126 126 215 215 215 F F F 22 440-459 124 124 124 207 207 207 F F F 23 460-479 123 123 123 206 206 206 F F F 24 480-499 126 126 126 211 211 211 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 126 126 126 211 211 211 T T T 27 540-559 121 121 121 203 203 203 T T T 28 560-579 126 126 126 214 214 214 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 126, Max flush: 215 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 205 205 205 F F F 1 020-039 124 124 124 207 207 207 F F F 2 040-059 124 124 124 209 209 209 F F F 3 060-079 126 126 126 212 212 212 F F F 4 080-099 125 125 125 210 210 210 F F F 5 100-119 124 124 124 208 208 208 F F F 6 120-139 127 127 127 216 216 216 F F F 7 140-159 120 120 120 201 201 201 F F F 8 160-179 122 122 122 204 204 204 F F F 9 180-199 126 126 126 212 212 212 F F F 10 200-219 127 127 127 216 216 216 F F F 11 220-239 127 127 127 216 216 216 F F F 12 240-259 126 126 126 212 212 212 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 126 126 126 211 211 211 F F F 15 300-319 126 126 126 212 212 212 F F F 16 320-339 126 126 126 213 213 213 F F F 17 340-359 126 126 126 213 213 213 F F F 18 360-379 123 123 123 206 206 206 F F F 19 380-399 124 124 124 209 209 209 F F F 20 400-419 126 126 126 215 215 215 F F F 21 420-439 126 126 126 215 215 215 F F F 22 440-459 124 124 124 207 207 207 F F F 23 460-479 123 123 123 206 206 206 F F F 24 480-499 126 126 126 211 211 211 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 127 127 127 216 216 216 F F F 27 540-559 127 127 127 216 216 216 F F F 28 560-579 126 126 126 214 214 214 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 127, Max flush: 216 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:22.999974166Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.044107777Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.044982035Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.0450215Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.045036978Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.045051386Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699} {"msg":"Crucible edf1454a-fbe1-44b8-8901-cb8317f4f8ca has session id: 7291552e-6800-4a8a-ad45-ebee1f97e885","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.045591234Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Upstairs opts: Upstairs UUID: edf1454a-fbe1-44b8-8901-cb8317f4f8ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.045637428Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Crucible stats registered with UUID: edf1454a-fbe1-44b8-8901-cb8317f4f8ca","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.045661165Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075400905Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.0754438Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699} {"msg":"edf1454a-fbe1-44b8-8901-cb8317f4f8ca active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075467967Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075484386Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"0","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075514162Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"0","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.07553157Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"1","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075549468Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"1","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075567876Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"2","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075586763Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"2","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075603732Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075621259Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075648396Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"ds_connection connected from Ok(127.0.0.1:48306)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075800129Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"ds_connection connected from Ok(127.0.0.1:40116)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075846633Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"ds_connection connected from Ok(127.0.0.1:46091)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.075968559Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.076760497Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"2","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.076808301Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"0","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.076942025Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"1","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.077477383Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"2","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.077604078Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"1","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.077685379Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"0","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078069914Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078235385Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"[0]R flush_numbers[0..12]: [205, 207, 209, 212, 210, 208, 216, 201, 204, 212, 216, 216]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078314985Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"[0]R generation[0..12]: [122, 124, 124, 126, 125, 124, 127, 120, 122, 126, 127, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078335813Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078350631Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"[1]R flush_numbers[0..12]: [205, 207, 209, 212, 210, 208, 216, 201, 204, 212, 216, 216]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078364729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"[1]R generation[0..12]: [122, 124, 124, 126, 125, 124, 127, 120, 122, 126, 127, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078378798Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078393046Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"[2]R flush_numbers[0..12]: [205, 207, 209, 212, 210, 208, 216, 201, 204, 212, 216, 216]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078406974Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"[2]R generation[0..12]: [122, 124, 124, 126, 125, 124, 127, 120, 122, 126, 127, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078424922Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078439471Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078453629Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078467098Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078480166Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078493134Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"":"downstairs","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078513742Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.07852606Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"0","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078539649Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"1","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078553907Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"client":"2","":"downstairs","session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078567216Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078579614Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"edf1454a-fbe1-44b8-8901-cb8317f4f8ca is now active with session: 7291552e-6800-4a8a-ad45-ebee1f97e885","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078592443Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.078604481Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699,"session_id":"7291552e-6800-4a8a-ad45-ebee1f97e885"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:23.07861694Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 109 len 4096 data: 10 02/30 Write block 88 len 16384 data: 14 18 15 16 03/30 Flush 04/30 Write block 73 len 28672 data: 7 8 6 2 3 5 4 05/30 Write block 22 len 4096 data: 13 06/30 Write block 572 len 28672 data: 17 14 13 12 10 10 9 07/30 Read block 281 len 32768 08/30 Read block 530 len 28672 09/30 Read block 223 len 8192 10/30 Read block 210 len 20480 11/30 Write block 533 len 16384 data: 13 13 11 12 12/30 Write block 60 len 32768 data: 4 4 5 6 7 7 10 13 13/30 Write block 340 len 24576 data: 9 11 13 13 12 12 14/30 Write block 238 len 20480 data: 9 7 6 7 8 15/30 Read block 308 len 4096 16/30 Read block 214 len 16384 17/30 Write block 480 len 16384 data: 12 10 8 6 18/30 Read block 374 len 36864 19/30 Write block 415 len 36864 data: 13 11 10 9 12 12 12 10 10 20/30 Read block 435 len 20480 21/30 Read block 6 len 36864 22/30 Read block 111 len 24576 23/30 Write block 53 len 36864 data: 11 14 14 10 10 9 8 5 5 24/30 Write block 384 len 12288 data: 11 9 9 25/30 Flush 26/30 Write block 80 len 12288 data: 5 5 10 27/30 Write block 238 len 36864 data: 10 8 7 8 9 6 6 6 6 28/30 Read block 386 len 16384 29/30 Read block 16 len 36864 30/30 Read block 467 len 8192 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:23.100185513Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1699} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 205 205 205 F F F 1 020-039 128 128 128 218 218 218 F F F 2 040-059 128 128 128 218 218 218 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 128 128 128 217 217 217 T T T 5 100-119 128 128 128 217 217 217 F F F 6 120-139 127 127 127 216 216 216 F F F 7 140-159 120 120 120 201 201 201 F F F 8 160-179 122 122 122 204 204 204 F F F 9 180-199 126 126 126 212 212 212 F F F 10 200-219 127 127 127 216 216 216 F F F 11 220-239 128 128 128 218 218 218 T T T 12 240-259 128 128 128 218 218 218 T T T 13 260-279 126 126 126 212 212 212 F F F 14 280-299 126 126 126 211 211 211 F F F 15 300-319 126 126 126 212 212 212 F F F 16 320-339 126 126 126 213 213 213 F F F 17 340-359 128 128 128 218 218 218 F F F 18 360-379 123 123 123 206 206 206 F F F 19 380-399 128 128 128 218 218 218 F F F 20 400-419 128 128 128 218 218 218 F F F 21 420-439 128 128 128 218 218 218 F F F 22 440-459 124 124 124 207 207 207 F F F 23 460-479 123 123 123 206 206 206 F F F 24 480-499 128 128 128 218 218 218 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 128 128 128 218 218 218 F F F 27 540-559 127 127 127 216 216 216 F F F 28 560-579 128 128 128 218 218 218 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 128, Max flush: 218 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 205 205 205 F F F 1 020-039 128 128 128 218 218 218 F F F 2 040-059 128 128 128 218 218 218 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 129 129 129 219 219 219 F F F 5 100-119 128 128 128 217 217 217 F F F 6 120-139 127 127 127 216 216 216 F F F 7 140-159 120 120 120 201 201 201 F F F 8 160-179 122 122 122 204 204 204 F F F 9 180-199 126 126 126 212 212 212 F F F 10 200-219 127 127 127 216 216 216 F F F 11 220-239 129 129 129 219 219 219 F F F 12 240-259 129 129 129 219 219 219 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 126 126 126 211 211 211 F F F 15 300-319 126 126 126 212 212 212 F F F 16 320-339 126 126 126 213 213 213 F F F 17 340-359 128 128 128 218 218 218 F F F 18 360-379 123 123 123 206 206 206 F F F 19 380-399 128 128 128 218 218 218 F F F 20 400-419 128 128 128 218 218 218 F F F 21 420-439 128 128 128 218 218 218 F F F 22 440-459 124 124 124 207 207 207 F F F 23 460-479 123 123 123 206 206 206 F F F 24 480-499 128 128 128 218 218 218 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 128 128 128 218 218 218 F F F 27 540-559 127 127 127 216 216 216 F F F 28 560-579 128 128 128 218 218 218 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 129, Max flush: 219 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.363301355Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.408759381Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.409686803Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.409727298Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.409741766Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.409757045Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707} {"msg":"Crucible cdf4fc9b-d372-416b-8edb-dd0aeac8d90d has session id: 34ec9dac-144f-4aa8-963c-58d0820b0204","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.410284563Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Upstairs opts: Upstairs UUID: cdf4fc9b-d372-416b-8edb-dd0aeac8d90d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.410333508Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Crucible stats registered with UUID: cdf4fc9b-d372-416b-8edb-dd0aeac8d90d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.410357774Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.441451197Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.44150879Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707} {"msg":"cdf4fc9b-d372-416b-8edb-dd0aeac8d90d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.441525868Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.441542146Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"0","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.441570692Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"0","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.44159598Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"1","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.441629286Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"1","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.441661942Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"2","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.441696528Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"2","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.441727534Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.441747032Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.44176402Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"ds_connection connected from Ok(127.0.0.1:55845)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.441803975Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"ds_connection connected from Ok(127.0.0.1:53337)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.441843971Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"ds_connection connected from Ok(127.0.0.1:63076)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.441883876Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.442819457Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"0","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.442854873Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"1","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.443128661Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"2","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.443591077Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"1","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.443758197Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"0","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.443792873Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"2","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444260998Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.44433452Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"[0]R flush_numbers[0..12]: [205, 218, 218, 218, 219, 217, 216, 201, 204, 212, 216, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444462515Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"[0]R generation[0..12]: [122, 128, 128, 128, 129, 128, 127, 120, 122, 126, 127, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444485023Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444500181Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"[1]R flush_numbers[0..12]: [205, 218, 218, 218, 219, 217, 216, 201, 204, 212, 216, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444520708Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"[1]R generation[0..12]: [122, 128, 128, 128, 129, 128, 127, 120, 122, 126, 127, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444539676Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444557254Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"[2]R flush_numbers[0..12]: [205, 218, 218, 218, 219, 217, 216, 201, 204, 212, 216, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444574682Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"[2]R generation[0..12]: [122, 128, 128, 128, 129, 128, 127, 120, 122, 126, 127, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.44459219Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444609888Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:56:27.444626166Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444648113Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444664411Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444680729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"":"downstairs","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444697398Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444712546Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"0","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444729404Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"1","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444747652Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"client":"2","":"downstairs","session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.44476469Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444782558Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"cdf4fc9b-d372-416b-8edb-dd0aeac8d90d is now active with session: 34ec9dac-144f-4aa8-963c-58d0820b0204","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444798696Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444813604Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707,"session_id":"34ec9dac-144f-4aa8-963c-58d0820b0204"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:27.444828842Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 224 len 4096 data: 11 02/30 Read block 467 len 12288 03/30 Read block 495 len 4096 04/30 Read block 375 len 40960 05/30 Flush 06/30 Write block 408 len 20480 data: 11 12 14 13 14 07/30 Write block 198 len 24576 data: 11 12 12 10 9 10 08/30 Read block 455 len 28672 09/30 Read block 32 len 32768 10/30 Read block 451 len 28672 11/30 Read block 53 len 36864 12/30 Read block 57 len 4096 13/30 Read block 288 len 36864 14/30 Read block 303 len 8192 15/30 Read block 456 len 28672 16/30 Read block 481 len 40960 17/30 Read block 47 len 8192 18/30 Read block 220 len 28672 19/30 Write block 328 len 4096 data: 12 20/30 Write block 323 len 28672 data: 11 10 13 11 11 13 9 21/30 Read block 347 len 8192 22/30 Read block 402 len 8192 23/30 Write block 148 len 36864 data: 13 13 11 11 10 10 8 8 8 24/30 Read block 507 len 40960 25/30 Read block 83 len 8192 26/30 Read block 95 len 16384 27/30 Read block 546 len 36864 28/30 Write block 21 len 32768 data: 14 14 16 16 15 16 15 13 29/30 Read block 503 len 8192 30/30 Write block 428 len 32768 data: 6 8 8 7 10 9 8 9 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 5 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:27.46515986Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1707} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 205 205 205 F F F 1 020-039 128 128 128 218 218 218 T T T 2 040-059 128 128 128 218 218 218 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 129 129 129 219 219 219 F F F 5 100-119 128 128 128 217 217 217 F F F 6 120-139 127 127 127 216 216 216 F F F 7 140-159 120 120 120 201 201 201 T T T 8 160-179 122 122 122 204 204 204 F F F 9 180-199 126 126 126 212 212 212 T T T 10 200-219 127 127 127 216 216 216 T T T 11 220-239 130 130 130 220 220 220 F F F 12 240-259 129 129 129 219 219 219 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 126 126 126 211 211 211 F F F 15 300-319 126 126 126 212 212 212 F F F 16 320-339 126 126 126 213 213 213 T T T 17 340-359 128 128 128 218 218 218 F F F 18 360-379 123 123 123 206 206 206 F F F 19 380-399 128 128 128 218 218 218 F F F 20 400-419 128 128 128 218 218 218 T T T 21 420-439 128 128 128 218 218 218 T T T 22 440-459 124 124 124 207 207 207 F F F 23 460-479 123 123 123 206 206 206 F F F 24 480-499 128 128 128 218 218 218 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 128 128 128 218 218 218 F F F 27 540-559 127 127 127 216 216 216 F F F 28 560-579 128 128 128 218 218 218 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 130, Max flush: 220 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 205 205 205 F F F 1 020-039 131 131 131 221 221 221 F F F 2 040-059 128 128 128 218 218 218 F F F 3 060-079 128 128 128 218 218 218 F F F 4 080-099 129 129 129 219 219 219 F F F 5 100-119 128 128 128 217 217 217 F F F 6 120-139 127 127 127 216 216 216 F F F 7 140-159 131 131 131 221 221 221 F F F 8 160-179 122 122 122 204 204 204 F F F 9 180-199 131 131 131 221 221 221 F F F 10 200-219 131 131 131 221 221 221 F F F 11 220-239 130 130 130 220 220 220 F F F 12 240-259 129 129 129 219 219 219 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 126 126 126 211 211 211 F F F 15 300-319 126 126 126 212 212 212 F F F 16 320-339 131 131 131 221 221 221 F F F 17 340-359 128 128 128 218 218 218 F F F 18 360-379 123 123 123 206 206 206 F F F 19 380-399 128 128 128 218 218 218 F F F 20 400-419 131 131 131 221 221 221 F F F 21 420-439 131 131 131 221 221 221 F F F 22 440-459 124 124 124 207 207 207 F F F 23 460-479 123 123 123 206 206 206 F F F 24 480-499 128 128 128 218 218 218 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 128 128 128 218 218 218 F F F 27 540-559 127 127 127 216 216 216 F F F 28 560-579 128 128 128 218 218 218 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 131, Max flush: 221 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.762004417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.806434982Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.80730881Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.807349425Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.807363963Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.807378282Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715} {"msg":"Crucible eb755ce1-3fcc-4258-843f-b70685b88959 has session id: fc77d97b-e288-4635-9b33-678afa8a9a14","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.807780515Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Upstairs opts: Upstairs UUID: eb755ce1-3fcc-4258-843f-b70685b88959, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.807826729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Crucible stats registered with UUID: eb755ce1-3fcc-4258-843f-b70685b88959","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.807850677Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838176158Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838234141Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715} {"msg":"eb755ce1-3fcc-4258-843f-b70685b88959 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838250019Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838266368Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"0","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838297174Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"0","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838321961Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"1","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838340339Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"1","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838369305Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"2","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838403761Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"2","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838434758Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838467454Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.83849803Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"ds_connection connected from Ok(127.0.0.1:37006)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838539876Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"ds_connection connected from Ok(127.0.0.1:39928)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838576781Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"ds_connection connected from Ok(127.0.0.1:33655)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.838610368Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.839460418Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"2","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.839495964Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"0","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.839630668Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"1","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.840145558Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"2","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.840181874Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"0","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.840203971Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"1","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.840737659Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.840850016Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"[0]R flush_numbers[0..12]: [205, 221, 218, 218, 219, 217, 216, 221, 204, 221, 221, 220]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.840923267Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"[0]R generation[0..12]: [122, 131, 128, 128, 129, 128, 127, 131, 122, 131, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.840945175Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.840960633Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"[1]R flush_numbers[0..12]: [205, 221, 218, 218, 219, 217, 216, 221, 204, 221, 221, 220]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.840974921Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"[1]R generation[0..12]: [122, 131, 128, 128, 129, 128, 127, 131, 122, 131, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.84098892Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841003588Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"[2]R flush_numbers[0..12]: [205, 221, 218, 218, 219, 217, 216, 221, 204, 221, 221, 220]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841017936Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"[2]R generation[0..12]: [122, 131, 128, 128, 129, 128, 127, 131, 122, 131, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841032565Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841046673Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841059722Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.84107679Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841090198Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841103357Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"":"downstairs","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:56:31.841116895Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841135813Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"0","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.84115806Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"1","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841174208Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"client":"2","":"downstairs","session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841187777Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841200235Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"eb755ce1-3fcc-4258-843f-b70685b88959 is now active with session: fc77d97b-e288-4635-9b33-678afa8a9a14","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841213034Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841225042Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715,"session_id":"fc77d97b-e288-4635-9b33-678afa8a9a14"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:31.841237191Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 350 len 20480 data: 9 8 7 4 7 02/30 Write block 346 len 8192 data: 13 11 03/30 Read block 51 len 12288 04/30 Read block 422 len 36864 05/30 Read block 10 len 12288 06/30 Write block 162 len 24576 data: 18 16 15 13 13 15 07/30 Write block 462 len 28672 data: 14 13 10 8 9 10 9 08/30 Read block 196 len 12288 09/30 Flush 10/30 Write block 439 len 20480 data: 9 7 6 6 10 11/30 Flush 12/30 Write block 445 len 12288 data: 7 8 7 13/30 Read block 548 len 4096 14/30 Read block 247 len 32768 15/30 Flush 16/30 Write block 401 len 4096 data: 11 17/30 Flush 18/30 Write block 59 len 20480 data: 9 6 6 6 7 19/30 Write block 364 len 28672 data: 9 9 9 9 7 5 5 20/30 Write block 246 len 20480 data: 7 6 7 5 6 21/30 Write block 487 len 24576 data: 11 11 11 8 7 8 22/30 Write block 285 len 28672 data: 6 7 8 7 9 11 12 23/30 Write block 14 len 40960 data: 10 10 15 14 15 16 13 15 15 17 24/30 Write block 93 len 32768 data: 16 12 12 12 11 11 9 7 25/30 Read block 169 len 24576 26/30 Write block 0 len 4096 data: 3 27/30 Write block 19 len 8192 data: 17 14 28/30 Write block 165 len 16384 data: 14 14 16 16 29/30 Write block 176 len 20480 data: 7 9 9 9 11 30/30 Write block 140 len 8192 data: 11 11 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 5 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:31.858847676Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1715} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 205 205 205 T T T 1 020-039 131 131 131 221 221 221 T T T 2 040-059 128 128 128 218 218 218 T T T 3 060-079 128 128 128 218 218 218 T T T 4 080-099 129 129 129 219 219 219 T T T 5 100-119 128 128 128 217 217 217 T T T 6 120-139 127 127 127 216 216 216 F F F 7 140-159 131 131 131 221 221 221 T T T 8 160-179 132 132 132 222 222 222 T T T 9 180-199 131 131 131 221 221 221 T T T 10 200-219 131 131 131 221 221 221 F F F 11 220-239 130 130 130 220 220 220 F F F 12 240-259 129 129 129 219 219 219 T T T 13 260-279 126 126 126 212 212 212 F F F 14 280-299 126 126 126 211 211 211 T T T 15 300-319 126 126 126 212 212 212 F F F 16 320-339 131 131 131 221 221 221 F F F 17 340-359 132 132 132 222 222 222 F F F 18 360-379 123 123 123 206 206 206 T T T 19 380-399 128 128 128 218 218 218 F F F 20 400-419 132 132 132 225 225 225 F F F 21 420-439 132 132 132 223 223 223 F F F 22 440-459 132 132 132 224 224 224 F F F 23 460-479 132 132 132 222 222 222 F F F 24 480-499 128 128 128 218 218 218 T T T 25 500-519 120 120 120 201 201 201 F F F 26 520-539 128 128 128 218 218 218 F F F 27 540-559 127 127 127 216 216 216 F F F 28 560-579 128 128 128 218 218 218 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 132, Max flush: 225 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 226 226 226 F F F 1 020-039 133 133 133 226 226 226 F F F 2 040-059 133 133 133 226 226 226 F F F 3 060-079 133 133 133 226 226 226 F F F 4 080-099 133 133 133 226 226 226 F F F 5 100-119 133 133 133 226 226 226 F F F 6 120-139 127 127 127 216 216 216 F F F 7 140-159 133 133 133 226 226 226 F F F 8 160-179 133 133 133 226 226 226 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 131 131 131 221 221 221 F F F 11 220-239 130 130 130 220 220 220 F F F 12 240-259 133 133 133 226 226 226 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 133 133 133 226 226 226 F F F 15 300-319 126 126 126 212 212 212 F F F 16 320-339 131 131 131 221 221 221 F F F 17 340-359 132 132 132 222 222 222 F F F 18 360-379 133 133 133 226 226 226 F F F 19 380-399 128 128 128 218 218 218 F F F 20 400-419 132 132 132 225 225 225 F F F 21 420-439 132 132 132 223 223 223 F F F 22 440-459 132 132 132 224 224 224 F F F 23 460-479 132 132 132 222 222 222 F F F 24 480-499 133 133 133 226 226 226 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 128 128 128 218 218 218 F F F 27 540-559 127 127 127 216 216 216 F F F 28 560-579 128 128 128 218 218 218 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 133, Max flush: 226 Begin loop 67 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.1794515Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.224251832Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.225140419Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.225181974Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.225197412Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.22521167Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723} {"msg":"Crucible 8e27c78a-1e4d-4d80-9f27-1de7eed4aafc has session id: 7b26fc8d-f496-431d-b883-0158f6ad645b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.225682576Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Upstairs opts: Upstairs UUID: 8e27c78a-1e4d-4d80-9f27-1de7eed4aafc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.22573001Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Crucible stats registered with UUID: 8e27c78a-1e4d-4d80-9f27-1de7eed4aafc","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.225759067Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.256573551Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.256636604Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723} {"msg":"8e27c78a-1e4d-4d80-9f27-1de7eed4aafc active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.256654322Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.25667114Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"0","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.256703826Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"0","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.256722063Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"1","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.25675102Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"1","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.256785206Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"2","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.256815413Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"2","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.256842209Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.256874726Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.256910262Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45851)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.256966865Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63415)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.257003541Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63276)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.257043846Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.257845343Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"0","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.257881218Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"1","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.258045139Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"2","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.258614263Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"1","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.258727569Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"2","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.258791552Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"0","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259209873Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259254698Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"[0]R flush_numbers[0..12]: [226, 226, 226, 226, 226, 226, 216, 226, 226, 226, 221, 220]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259429788Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"[0]R generation[0..12]: [133, 133, 133, 133, 133, 133, 127, 133, 133, 133, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259450265Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259465443Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"[1]R flush_numbers[0..12]: [226, 226, 226, 226, 226, 226, 216, 226, 226, 226, 221, 220]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259479742Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"[1]R generation[0..12]: [133, 133, 133, 133, 133, 133, 127, 133, 133, 133, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.25949365Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259512098Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"[2]R flush_numbers[0..12]: [226, 226, 226, 226, 226, 226, 216, 226, 226, 226, 221, 220]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259526486Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"[2]R generation[0..12]: [133, 133, 133, 133, 133, 133, 127, 133, 133, 133, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259540335Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259554213Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259567971Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.25958129Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259594308Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-01-13T15:56:36.259607267Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"":"downstairs","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259627484Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259639723Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"0","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259653252Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"1","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.25966628Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"client":"2","":"downstairs","session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259679709Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259691967Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"8e27c78a-1e4d-4d80-9f27-1de7eed4aafc is now active with session: 7b26fc8d-f496-431d-b883-0158f6ad645b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259704906Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259720374Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723,"session_id":"7b26fc8d-f496-431d-b883-0158f6ad645b"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:36.259733082Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 286 len 40960 data: 8 9 8 10 12 13 14 12 12 8 02/30 Read block 435 len 40960 03/30 Write block 406 len 4096 data: 13 04/30 Read block 551 len 4096 05/30 Read block 221 len 4096 06/30 Read block 65 len 24576 07/30 Read block 114 len 20480 08/30 Flush 09/30 Write block 145 len 4096 data: 10 10/30 Read block 177 len 8192 11/30 Read block 169 len 8192 12/30 Read block 331 len 32768 13/30 Read block 372 len 32768 14/30 Read block 482 len 36864 15/30 Flush 16/30 Write block 329 len 40960 data: 10 9 7 8 7 6 5 8 9 10 17/30 Read block 115 len 16384 18/30 Read block 132 len 24576 19/30 Read block 444 len 20480 20/30 Read block 561 len 36864 21/30 Read block 195 len 36864 22/30 Read block 405 len 40960 23/30 Write block 313 len 4096 data: 6 24/30 Read block 440 len 36864 25/30 Write block 66 len 40960 data: 11 14 13 11 12 10 8 8 9 7 26/30 Read block 230 len 16384 27/30 Read block 300 len 8192 28/30 Read block 84 len 32768 29/30 Write block 76 len 28672 data: 3 4 6 5 6 6 11 30/30 Write block 407 len 28672 data: 9 12 13 15 14 15 15 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 10 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:36.280931259Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1723} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 226 226 226 F F F 1 020-039 133 133 133 226 226 226 F F F 2 040-059 133 133 133 226 226 226 F F F 3 060-079 133 133 133 226 226 226 T T T 4 080-099 133 133 133 226 226 226 T T T 5 100-119 133 133 133 226 226 226 F F F 6 120-139 127 127 127 216 216 216 F F F 7 140-159 134 134 134 228 228 228 F F F 8 160-179 133 133 133 226 226 226 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 131 131 131 221 221 221 F F F 11 220-239 130 130 130 220 220 220 F F F 12 240-259 133 133 133 226 226 226 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 134 134 134 227 227 227 F F F 15 300-319 126 126 126 212 212 212 T T T 16 320-339 131 131 131 221 221 221 T T T 17 340-359 132 132 132 222 222 222 F F F 18 360-379 133 133 133 226 226 226 F F F 19 380-399 128 128 128 218 218 218 F F F 20 400-419 134 134 134 227 227 227 T T T 21 420-439 132 132 132 223 223 223 F F F 22 440-459 132 132 132 224 224 224 F F F 23 460-479 132 132 132 222 222 222 F F F 24 480-499 133 133 133 226 226 226 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 128 128 128 218 218 218 F F F 27 540-559 127 127 127 216 216 216 F F F 28 560-579 128 128 128 218 218 218 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 134, Max flush: 228 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 226 226 226 F F F 1 020-039 133 133 133 226 226 226 F F F 2 040-059 133 133 133 226 226 226 F F F 3 060-079 135 135 135 229 229 229 F F F 4 080-099 135 135 135 229 229 229 F F F 5 100-119 133 133 133 226 226 226 F F F 6 120-139 127 127 127 216 216 216 F F F 7 140-159 134 134 134 228 228 228 F F F 8 160-179 133 133 133 226 226 226 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 131 131 131 221 221 221 F F F 11 220-239 130 130 130 220 220 220 F F F 12 240-259 133 133 133 226 226 226 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 134 134 134 227 227 227 F F F 15 300-319 135 135 135 229 229 229 F F F 16 320-339 135 135 135 229 229 229 F F F 17 340-359 132 132 132 222 222 222 F F F 18 360-379 133 133 133 226 226 226 F F F 19 380-399 128 128 128 218 218 218 F F F 20 400-419 135 135 135 229 229 229 F F F 21 420-439 132 132 132 223 223 223 F F F 22 440-459 132 132 132 224 224 224 F F F 23 460-479 132 132 132 222 222 222 F F F 24 480-499 133 133 133 226 226 226 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 128 128 128 218 218 218 F F F 27 540-559 127 127 127 216 216 216 F F F 28 560-579 128 128 128 218 218 218 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 135, Max flush: 229 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.563312273Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:40.606228835Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.606607841Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.638140222Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.6387502Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.638783166Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.638796615Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.638810613Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731} {"msg":"Crucible 2594479f-dcd0-40eb-96ce-ab6d7d1d79ab has session id: e8a9f9d5-285d-427a-8ba5-b10787be64e2","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.639304466Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"Upstairs opts: Upstairs UUID: 2594479f-dcd0-40eb-96ce-ab6d7d1d79ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.639359909Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"Crucible stats registered with UUID: 2594479f-dcd0-40eb-96ce-ab6d7d1d79ab","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.639389626Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.669837833Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.669892656Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731} {"msg":"2594479f-dcd0-40eb-96ce-ab6d7d1d79ab active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.669921263Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.669940611Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"0","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.669984296Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"0","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.670019251Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"1","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.670053578Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"1","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.670092803Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"2","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.670126119Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"2","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.670154246Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.670180393Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.67020406Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:55230)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.670228577Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:55814)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.670269742Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:39021)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.670455381Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.671007396Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"1","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.67106028Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"2","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.671328019Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"0","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.671837599Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"1","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.671867576Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"2","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.671951456Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"0","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672454127Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672481584Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"[0]R flush_numbers[0..12]: [226, 226, 226, 229, 229, 226, 216, 228, 226, 226, 221, 220]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672513291Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"[0]R generation[0..12]: [133, 133, 133, 135, 135, 133, 127, 134, 133, 133, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672529249Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672543767Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"[1]R flush_numbers[0..12]: [226, 226, 226, 229, 229, 226, 216, 228, 226, 226, 221, 220]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672557655Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"[1]R generation[0..12]: [133, 133, 133, 135, 135, 133, 127, 134, 133, 133, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672571764Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672586182Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"[2]R flush_numbers[0..12]: [226, 226, 226, 229, 229, 226, 216, 228, 226, 226, 221, 220]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672600001Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"[2]R generation[0..12]: [133, 133, 133, 135, 135, 133, 127, 134, 133, 133, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672613829Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672627897Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672641066Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672654394Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"Next flush: 230","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672667293Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672683601Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"":"downstairs","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672697289Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672709628Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"0","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672729495Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"1","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672742804Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"client":"2","":"downstairs","session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672755962Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672768321Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"2594479f-dcd0-40eb-96ce-ab6d7d1d79ab is now active with session: e8a9f9d5-285d-427a-8ba5-b10787be64e2","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672781109Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672793238Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731,"session_id":"e8a9f9d5-285d-427a-8ba5-b10787be64e2"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:40.672805566Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 426 len 12288 data: 8 6 7 02/30 Write block 151 len 40960 data: 12 11 11 9 9 9 9 11 11 14 03/30 Write block 311 len 16384 data: 5 5 7 7 04/30 Read block 246 len 32768 05/30 Write block 344 len 36864 data: 13 13 14 12 11 8 10 9 8 06/30 Read block 396 len 32768 07/30 Read block 563 len 28672 08/30 Read block 488 len 12288 09/30 Write block 536 len 32768 data: 13 15 14 14 12 11 10 10 10/30 Flush 11/30 Write block 160 len 24576 data: 15 16 19 17 16 15 12/30 Flush 13/30 Write block 399 len 32768 data: 12 11 12 14 16 14 13 14 14/30 Read block 451 len 36864 15/30 Write block 10 len 24576 data: 10 10 11 11 11 11 16/30 Read block 525 len 40960 17/30 Read block 7 len 36864 18/30 Read block 435 len 4096 19/30 Write block 227 len 40960 data: 9 13 11 14 15 15 9 9 8 6 20/30 Read block 3 len 12288 21/30 Read block 99 len 40960 22/30 Read block 207 len 20480 23/30 Write block 135 len 36864 data: 11 11 10 9 8 12 12 10 10 24/30 Read block 125 len 12288 25/30 Write block 306 len 4096 data: 8 26/30 Read block 115 len 24576 27/30 Read block 30 len 32768 28/30 Read block 545 len 16384 29/30 Read block 346 len 16384 30/30 Read block 428 len 12288 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 8 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 18 18 17 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:40.694259453Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1731} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 226 226 226 T T T 1 020-039 133 133 133 226 226 226 F F F 2 040-059 133 133 133 226 226 226 F F F 3 060-079 135 135 135 229 229 229 F F F 4 080-099 135 135 135 229 229 229 F F F 5 100-119 133 133 133 226 226 226 F F F 6 120-139 127 127 127 216 216 216 T T T 7 140-159 136 136 136 230 230 230 T T T 8 160-179 136 136 136 231 231 231 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 131 131 131 221 221 221 F F F 11 220-239 130 130 130 220 220 220 T T T 12 240-259 133 133 133 226 226 226 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 134 134 134 227 227 227 F F F 15 300-319 136 136 136 230 230 230 T T T 16 320-339 135 135 135 229 229 229 F F F 17 340-359 136 136 136 230 230 230 F F F 18 360-379 133 133 133 226 226 226 F F F 19 380-399 128 128 128 218 218 218 T T T 20 400-419 135 135 135 229 229 229 T T T 21 420-439 136 136 136 230 230 230 F F F 22 440-459 132 132 132 224 224 224 F F F 23 460-479 132 132 132 222 222 222 F F F 24 480-499 133 133 133 226 226 226 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 136 136 136 230 230 230 F F F 27 540-559 136 136 136 230 230 230 F F F 28 560-579 128 128 128 218 218 218 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 136, Max flush: 231 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 232 232 232 F F F 1 020-039 133 133 133 226 226 226 F F F 2 040-059 133 133 133 226 226 226 F F F 3 060-079 135 135 135 229 229 229 F F F 4 080-099 135 135 135 229 229 229 F F F 5 100-119 133 133 133 226 226 226 F F F 6 120-139 137 137 137 232 232 232 F F F 7 140-159 137 137 137 232 232 232 F F F 8 160-179 136 136 136 231 231 231 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 131 131 131 221 221 221 F F F 11 220-239 137 137 137 232 232 232 F F F 12 240-259 133 133 133 226 226 226 F F F 13 260-279 126 126 126 212 212 212 F F F 14 280-299 134 134 134 227 227 227 F F F 15 300-319 137 137 137 232 232 232 F F F 16 320-339 135 135 135 229 229 229 F F F 17 340-359 136 136 136 230 230 230 F F F 18 360-379 133 133 133 226 226 226 F F F 19 380-399 137 137 137 232 232 232 F F F 20 400-419 137 137 137 232 232 232 F F F 21 420-439 136 136 136 230 230 230 F F F 22 440-459 132 132 132 224 224 224 F F F 23 460-479 132 132 132 222 222 222 F F F 24 480-499 133 133 133 226 226 226 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 136 136 136 230 230 230 F F F 27 540-559 136 136 136 230 230 230 F F F 28 560-579 128 128 128 218 218 218 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 137, Max flush: 232 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.010206381Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.055449111Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.056359285Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.056408119Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.056425977Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.056444075Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739} {"msg":"Crucible e1a40c41-e470-4512-8508-9e2beee599f7 has session id: 81a00832-e64b-4d04-ba5c-978c9f7cea2e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.05692168Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Upstairs opts: Upstairs UUID: e1a40c41-e470-4512-8508-9e2beee599f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.056986032Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Crucible stats registered with UUID: e1a40c41-e470-4512-8508-9e2beee599f7","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.057010759Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.087882737Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.08793895Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739} {"msg":"e1a40c41-e470-4512-8508-9e2beee599f7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.087955338Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.087971927Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"0","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.088000383Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"0","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.088018511Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"1","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.088037069Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"1","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.088066336Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"2","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.088100812Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"2","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.088209719Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.088253214Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.08828583Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38938)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.088320756Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41934)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.088415615Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48123)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.0884588Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.089128252Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"0","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.089175246Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"2","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.089407109Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"1","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.089905121Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"2","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.089935138Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"0","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.089952445Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"1","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.0905104Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090547666Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"[0]R flush_numbers[0..12]: [232, 226, 226, 229, 229, 226, 232, 232, 231, 226, 221, 232]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090578012Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"[0]R generation[0..12]: [137, 133, 133, 135, 135, 133, 137, 137, 136, 133, 131, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.09059523Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090609939Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"[1]R flush_numbers[0..12]: [232, 226, 226, 229, 229, 226, 232, 232, 231, 226, 221, 232]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090624017Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"[1]R generation[0..12]: [137, 133, 133, 135, 135, 133, 137, 137, 136, 133, 131, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090638275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090652674Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"[2]R flush_numbers[0..12]: [232, 226, 226, 229, 229, 226, 232, 232, 231, 226, 221, 232]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090670692Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"[2]R generation[0..12]: [137, 133, 133, 135, 135, 133, 137, 137, 136, 133, 131, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.09068515Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090699308Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090712487Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090726025Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090739224Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090752432Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"":"downstairs","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090775719Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:56:45.090799597Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"0","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090831793Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"1","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090847301Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"client":"2","":"downstairs","session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090860859Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090873218Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"e1a40c41-e470-4512-8508-9e2beee599f7 is now active with session: 81a00832-e64b-4d04-ba5c-978c9f7cea2e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090886116Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090905814Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739,"session_id":"81a00832-e64b-4d04-ba5c-978c9f7cea2e"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:45.090933731Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 413 len 4096 data: 16 02/30 Read block 292 len 8192 03/30 Write block 476 len 32768 data: 9 8 11 12 13 11 9 7 04/30 Write block 248 len 12288 data: 8 6 7 05/30 Read block 493 len 36864 06/30 Write block 42 len 24576 data: 11 14 13 11 12 10 07/30 Flush 08/30 Write block 126 len 28672 data: 13 14 12 8 5 6 6 09/30 Write block 493 len 20480 data: 7 9 8 9 10 10/30 Write block 97 len 4096 data: 12 11/30 Write block 523 len 8192 data: 9 8 12/30 Read block 408 len 24576 13/30 Flush 14/30 Write block 358 len 24576 data: 9 12 10 9 10 9 15/30 Write block 42 len 20480 data: 12 15 14 12 13 16/30 Read block 142 len 24576 17/30 Write block 542 len 32768 data: 11 11 12 12 14 12 10 10 18/30 Read block 443 len 40960 19/30 Write block 126 len 16384 data: 14 15 13 9 20/30 Write block 394 len 16384 data: 5 4 4 12 21/30 Write block 593 len 16384 data: 14 15 12 11 22/30 Read block 17 len 32768 23/30 Read block 570 len 12288 24/30 Read block 30 len 32768 25/30 Write block 205 len 12288 data: 11 12 12 26/30 Write block 449 len 4096 data: 10 27/30 Read block 503 len 8192 28/30 Write block 263 len 4096 data: 14 29/30 Read block 11 len 12288 30/30 Write block 151 len 20480 data: 13 12 12 10 10 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 6 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:45.108928021Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1739} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 232 232 232 F F F 1 020-039 133 133 133 226 226 226 F F F 2 040-059 138 138 138 233 233 233 T T T 3 060-079 135 135 135 229 229 229 F F F 4 080-099 138 138 138 234 234 234 F F F 5 100-119 133 133 133 226 226 226 F F F 6 120-139 138 138 138 234 234 234 T T T 7 140-159 137 137 137 232 232 232 T T T 8 160-179 136 136 136 231 231 231 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 131 131 131 221 221 221 T T T 11 220-239 137 137 137 232 232 232 F F F 12 240-259 138 138 138 233 233 233 F F F 13 260-279 126 126 126 212 212 212 T T T 14 280-299 134 134 134 227 227 227 F F F 15 300-319 137 137 137 232 232 232 F F F 16 320-339 135 135 135 229 229 229 F F F 17 340-359 136 136 136 230 230 230 T T T 18 360-379 133 133 133 226 226 226 T T T 19 380-399 137 137 137 232 232 232 T T T 20 400-419 138 138 138 233 233 233 F F F 21 420-439 136 136 136 230 230 230 F F F 22 440-459 132 132 132 224 224 224 T T T 23 460-479 138 138 138 233 233 233 F F F 24 480-499 138 138 138 234 234 234 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 138 138 138 234 234 234 F F F 27 540-559 136 136 136 230 230 230 T T T 28 560-579 128 128 128 218 218 218 F F F 29 580-599 126 126 126 213 213 213 T T T Max gen: 138, Max flush: 234 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 232 232 232 F F F 1 020-039 133 133 133 226 226 226 F F F 2 040-059 139 139 139 235 235 235 F F F 3 060-079 135 135 135 229 229 229 F F F 4 080-099 138 138 138 234 234 234 F F F 5 100-119 133 133 133 226 226 226 F F F 6 120-139 139 139 139 235 235 235 F F F 7 140-159 139 139 139 235 235 235 F F F 8 160-179 136 136 136 231 231 231 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 139 139 139 235 235 235 F F F 11 220-239 137 137 137 232 232 232 F F F 12 240-259 138 138 138 233 233 233 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 134 134 134 227 227 227 F F F 15 300-319 137 137 137 232 232 232 F F F 16 320-339 135 135 135 229 229 229 F F F 17 340-359 139 139 139 235 235 235 F F F 18 360-379 139 139 139 235 235 235 F F F 19 380-399 139 139 139 235 235 235 F F F 20 400-419 138 138 138 233 233 233 F F F 21 420-439 136 136 136 230 230 230 F F F 22 440-459 139 139 139 235 235 235 F F F 23 460-479 138 138 138 233 233 233 F F F 24 480-499 138 138 138 234 234 234 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 138 138 138 234 234 234 F F F 27 540-559 139 139 139 235 235 235 F F F 28 560-579 128 128 128 218 218 218 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 139, Max flush: 235 Begin loop 70 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.425510966Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:49.470166086Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.470736089Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.503667786Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.504367445Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.504413239Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.504430797Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.504448735Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747} {"msg":"Crucible 141f12e9-8494-48c8-893b-26a82b49b722 has session id: b607b724-2a9b-4b61-9d81-60a11eedc284","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.504906582Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Upstairs opts: Upstairs UUID: 141f12e9-8494-48c8-893b-26a82b49b722, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.504964905Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Crucible stats registered with UUID: 141f12e9-8494-48c8-893b-26a82b49b722","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.504987463Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535362198Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535412592Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747} {"msg":"141f12e9-8494-48c8-893b-26a82b49b722 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.53542924Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535445098Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"0","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535474775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"0","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535492453Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"1","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535510231Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"1","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535535468Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"2","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535570464Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"2","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.53560382Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535635096Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535667243Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"ds_connection connected from Ok(127.0.0.1:56890)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535748623Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"ds_connection connected from Ok(127.0.0.1:33006)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535793228Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"ds_connection connected from Ok(127.0.0.1:61649)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.535828464Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.536540991Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"1","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.536586875Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"2","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.536732758Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"0","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.537309751Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"1","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.537349296Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"2","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.537366494Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"0","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.537896132Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.537923859Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"[0]R flush_numbers[0..12]: [232, 226, 235, 229, 234, 226, 235, 235, 231, 226, 235, 232]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.537987422Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"[0]R generation[0..12]: [137, 133, 139, 135, 138, 133, 139, 139, 136, 133, 139, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538008549Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538023688Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"[1]R flush_numbers[0..12]: [232, 226, 235, 229, 234, 226, 235, 235, 231, 226, 235, 232]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538037926Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"[1]R generation[0..12]: [137, 133, 139, 135, 138, 133, 139, 139, 136, 133, 139, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538052094Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538066283Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"[2]R flush_numbers[0..12]: [232, 226, 235, 229, 234, 226, 235, 235, 231, 226, 235, 232]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538080101Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"[2]R generation[0..12]: [137, 133, 139, 135, 138, 133, 139, 139, 136, 133, 139, 137]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538093849Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538108028Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538121356Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538134815Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538148483Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538165651Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"":"downstairs","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538185979Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538198417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"0","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538212026Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"1","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538225144Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"client":"2","":"downstairs","session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538238323Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538250681Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"141f12e9-8494-48c8-893b-26a82b49b722 is now active with session: b607b724-2a9b-4b61-9d81-60a11eedc284","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.53826363Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538275748Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747,"session_id":"b607b724-2a9b-4b61-9d81-60a11eedc284"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:49.538287997Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 59 len 40960 data: 10 7 7 7 8 8 8 12 15 14 02/30 Flush 03/30 Write block 121 len 28672 data: 11 12 14 13 12 15 16 04/30 Read block 214 len 16384 05/30 Read block 353 len 24576 06/30 Write block 558 len 36864 data: 8 9 8 9 11 11 12 13 14 07/30 Read block 343 len 28672 08/30 Read block 235 len 8192 09/30 Write block 424 len 28672 data: 10 9 9 7 8 9 9 10/30 Write block 232 len 28672 data: 16 10 10 9 7 8 11 11/30 Read block 235 len 12288 12/30 Read block 505 len 32768 13/30 Read block 583 len 36864 14/30 Read block 367 len 28672 15/30 Write block 114 len 40960 data: 6 6 13 14 13 14 13 12 13 15 16/30 Write block 326 len 40960 data: 12 12 14 11 10 8 9 8 7 6 17/30 Write block 493 len 24576 data: 8 10 9 10 11 12 18/30 Read block 428 len 16384 19/30 Write block 91 len 32768 data: 17 15 17 13 13 13 13 12 20/30 Read block 436 len 40960 21/30 Write block 479 len 40960 data: 13 14 12 10 8 7 9 11 12 12 22/30 Write block 365 len 8192 data: 10 10 23/30 Write block 25 len 12288 data: 16 17 16 24/30 Read block 180 len 8192 25/30 Read block 331 len 12288 26/30 Write block 547 len 8192 data: 13 11 27/30 Read block 302 len 4096 28/30 Write block 572 len 20480 data: 18 15 14 13 11 29/30 Read block 393 len 8192 30/30 Write block 403 len 40960 data: 17 15 14 15 10 13 14 16 15 16 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 7 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:49.558762018Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1747} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 232 232 232 F F F 1 020-039 133 133 133 226 226 226 T T T 2 040-059 140 140 140 236 236 236 F F F 3 060-079 140 140 140 236 236 236 F F F 4 080-099 138 138 138 234 234 234 T T T 5 100-119 133 133 133 226 226 226 T T T 6 120-139 139 139 139 235 235 235 T T T 7 140-159 139 139 139 235 235 235 F F F 8 160-179 136 136 136 231 231 231 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 139 139 139 235 235 235 F F F 11 220-239 137 137 137 232 232 232 T T T 12 240-259 138 138 138 233 233 233 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 134 134 134 227 227 227 F F F 15 300-319 137 137 137 232 232 232 F F F 16 320-339 135 135 135 229 229 229 T T T 17 340-359 139 139 139 235 235 235 F F F 18 360-379 139 139 139 235 235 235 T T T 19 380-399 139 139 139 235 235 235 F F F 20 400-419 138 138 138 233 233 233 T T T 21 420-439 136 136 136 230 230 230 T T T 22 440-459 139 139 139 235 235 235 F F F 23 460-479 138 138 138 233 233 233 T T T 24 480-499 138 138 138 234 234 234 T T T 25 500-519 120 120 120 201 201 201 F F F 26 520-539 138 138 138 234 234 234 F F F 27 540-559 139 139 139 235 235 235 T T T 28 560-579 128 128 128 218 218 218 T T T 29 580-599 139 139 139 235 235 235 F F F Max gen: 140, Max flush: 236 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 232 232 232 F F F 1 020-039 141 141 141 237 237 237 F F F 2 040-059 140 140 140 236 236 236 F F F 3 060-079 140 140 140 236 236 236 F F F 4 080-099 141 141 141 237 237 237 F F F 5 100-119 141 141 141 237 237 237 F F F 6 120-139 141 141 141 237 237 237 F F F 7 140-159 139 139 139 235 235 235 F F F 8 160-179 136 136 136 231 231 231 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 139 139 139 235 235 235 F F F 11 220-239 141 141 141 237 237 237 F F F 12 240-259 138 138 138 233 233 233 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 134 134 134 227 227 227 F F F 15 300-319 137 137 137 232 232 232 F F F 16 320-339 141 141 141 237 237 237 F F F 17 340-359 139 139 139 235 235 235 F F F 18 360-379 141 141 141 237 237 237 F F F 19 380-399 139 139 139 235 235 235 F F F 20 400-419 141 141 141 237 237 237 F F F 21 420-439 141 141 141 237 237 237 F F F 22 440-459 139 139 139 235 235 235 F F F 23 460-479 141 141 141 237 237 237 F F F 24 480-499 141 141 141 237 237 237 F F F 25 500-519 120 120 120 201 201 201 F F F 26 520-539 138 138 138 234 234 234 F F F 27 540-559 141 141 141 237 237 237 F F F 28 560-579 141 141 141 237 237 237 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 141, Max flush: 237 Begin loop 71 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.914283336Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.960900196Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.961773964Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.961843046Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.961871343Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.961901439Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755} {"msg":"Crucible a228bd12-12a8-471b-8e2d-b0dad2e6c518 has session id: ad2f722a-07f7-4896-b763-d0e9918aa99b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.962364185Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Upstairs opts: Upstairs UUID: a228bd12-12a8-471b-8e2d-b0dad2e6c518, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.962411909Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Crucible stats registered with UUID: a228bd12-12a8-471b-8e2d-b0dad2e6c518","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.962429937Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.99308787Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.993143354Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755} {"msg":"a228bd12-12a8-471b-8e2d-b0dad2e6c518 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.993159262Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.99317549Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"0","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.993205146Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"0","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.993224404Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"1","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.993242252Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"1","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.99325873Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"2","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.993276548Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"2","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.993293056Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.993310234Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.993327592Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63045)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.993505001Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35824)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.993841262Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47742)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.993880657Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.994328055Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"1","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.99437149Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"0","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.994403636Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"2","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995020584Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"1","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995067009Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"2","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995105035Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"0","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995622864Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995726012Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"[0]R flush_numbers[0..12]: [232, 237, 236, 236, 237, 237, 237, 235, 231, 226, 235, 237]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.99582657Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"[0]R generation[0..12]: [137, 141, 140, 140, 141, 141, 141, 139, 136, 133, 139, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995858137Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995874785Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"[1]R flush_numbers[0..12]: [232, 237, 236, 236, 237, 237, 237, 235, 231, 226, 235, 237]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995890203Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"[1]R generation[0..12]: [137, 141, 140, 140, 141, 141, 141, 139, 136, 133, 139, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995908761Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995923589Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"[2]R flush_numbers[0..12]: [232, 237, 236, 236, 237, 237, 237, 235, 231, 226, 235, 237]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995937797Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"[2]R generation[0..12]: [137, 141, 140, 140, 141, 141, 141, 139, 136, 133, 139, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995952566Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995966604Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995979763Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.995993261Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.99600648Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:56:53.996019928Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"":"downstairs","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.996039466Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.996051624Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"0","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.996065123Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"1","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.996078701Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"client":"2","":"downstairs","session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.99609249Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.996108897Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"a228bd12-12a8-471b-8e2d-b0dad2e6c518 is now active with session: ad2f722a-07f7-4896-b763-d0e9918aa99b","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.996123186Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.996135525Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755,"session_id":"ad2f722a-07f7-4896-b763-d0e9918aa99b"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:53.996148073Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 96 len 32768 data: 14 14 13 10 8 8 8 9 02/30 Read block 435 len 36864 03/30 Read block 122 len 4096 04/30 Write block 452 len 20480 data: 10 10 9 11 10 05/30 Read block 238 len 32768 06/30 Write block 419 len 40960 data: 13 13 13 11 11 11 10 10 8 9 07/30 Read block 494 len 20480 08/30 Write block 229 len 4096 data: 12 09/30 Flush 10/30 Write block 555 len 4096 data: 8 11/30 Flush 12/30 Write block 407 len 24576 data: 11 14 15 17 16 17 13/30 Write block 550 len 20480 data: 9 12 10 10 6 14/30 Read block 91 len 32768 15/30 Write block 576 len 16384 data: 12 11 10 7 16/30 Write block 417 len 32768 data: 11 10 14 14 14 12 12 12 17/30 Write block 154 len 24576 data: 11 11 10 10 12 12 18/30 Write block 314 len 32768 data: 8 7 7 6 7 5 10 10 19/30 Read block 262 len 4096 20/30 Write block 394 len 16384 data: 6 5 5 13 21/30 Write block 473 len 40960 data: 6 8 8 10 9 12 14 15 13 11 22/30 Read block 310 len 16384 23/30 Write block 501 len 32768 data: 15 15 15 13 10 9 8 9 24/30 Write block 127 len 36864 data: 17 14 10 6 7 7 7 9 12 25/30 Write block 102 len 8192 data: 9 10 26/30 Read block 415 len 8192 27/30 Write block 150 len 32768 data: 12 14 13 13 12 12 11 11 28/30 Write block 456 len 36864 data: 11 11 12 15 14 14 15 14 11 29/30 Write block 413 len 20480 data: 17 14 14 12 12 30/30 Write block 317 len 16384 data: 7 8 6 11 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 6 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:54.015216888Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1755} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 232 232 232 F F F 1 020-039 141 141 141 237 237 237 F F F 2 040-059 140 140 140 236 236 236 F F F 3 060-079 140 140 140 236 236 236 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 142 142 142 238 238 238 T T T 6 120-139 141 141 141 237 237 237 T T T 7 140-159 139 139 139 235 235 235 T T T 8 160-179 136 136 136 231 231 231 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 139 139 139 235 235 235 F F F 11 220-239 142 142 142 238 238 238 F F F 12 240-259 138 138 138 233 233 233 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 134 134 134 227 227 227 F F F 15 300-319 137 137 137 232 232 232 T T T 16 320-339 141 141 141 237 237 237 T T T 17 340-359 139 139 139 235 235 235 F F F 18 360-379 141 141 141 237 237 237 F F F 19 380-399 139 139 139 235 235 235 T T T 20 400-419 142 142 142 238 238 238 T T T 21 420-439 142 142 142 238 238 238 T T T 22 440-459 142 142 142 238 238 238 T T T 23 460-479 141 141 141 237 237 237 T T T 24 480-499 141 141 141 237 237 237 T T T 25 500-519 120 120 120 201 201 201 T T T 26 520-539 138 138 138 234 234 234 F F F 27 540-559 142 142 142 239 239 239 T T T 28 560-579 141 141 141 237 237 237 T T T 29 580-599 139 139 139 235 235 235 F F F Max gen: 142, Max flush: 239 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 232 232 232 F F F 1 020-039 141 141 141 237 237 237 F F F 2 040-059 140 140 140 236 236 236 F F F 3 060-079 140 140 140 236 236 236 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 143 143 143 240 240 240 F F F 6 120-139 143 143 143 240 240 240 F F F 7 140-159 143 143 143 240 240 240 F F F 8 160-179 136 136 136 231 231 231 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 139 139 139 235 235 235 F F F 11 220-239 142 142 142 238 238 238 F F F 12 240-259 138 138 138 233 233 233 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 134 134 134 227 227 227 F F F 15 300-319 143 143 143 240 240 240 F F F 16 320-339 143 143 143 240 240 240 F F F 17 340-359 139 139 139 235 235 235 F F F 18 360-379 141 141 141 237 237 237 F F F 19 380-399 143 143 143 240 240 240 F F F 20 400-419 143 143 143 240 240 240 F F F 21 420-439 143 143 143 240 240 240 F F F 22 440-459 143 143 143 240 240 240 F F F 23 460-479 143 143 143 240 240 240 F F F 24 480-499 143 143 143 240 240 240 F F F 25 500-519 143 143 143 240 240 240 F F F 26 520-539 138 138 138 234 234 234 F F F 27 540-559 143 143 143 240 240 240 F F F 28 560-579 143 143 143 240 240 240 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 143, Max flush: 240 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.349271543Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.393281608Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.394046719Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.394092924Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.394111001Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.394129539Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763} {"msg":"Crucible 23a1663e-8007-471f-8d3f-140fd50bfd95 has session id: 0108eedf-4f27-4c7a-9f95-4117e6a1af25","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.394498966Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Upstairs opts: Upstairs UUID: 23a1663e-8007-471f-8d3f-140fd50bfd95, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.39455026Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Crucible stats registered with UUID: 23a1663e-8007-471f-8d3f-140fd50bfd95","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.394576957Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.424578476Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.42463456Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763} {"msg":"23a1663e-8007-471f-8d3f-140fd50bfd95 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.424667136Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.424697012Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"0","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.424743807Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"0","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.424789682Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"1","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.424827387Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"1","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.424861433Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"2","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.424895299Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"2","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.424930405Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.424968541Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.425002347Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"ds_connection connected from Ok(127.0.0.1:62241)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.425037922Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"ds_connection connected from Ok(127.0.0.1:40565)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.425092236Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"ds_connection connected from Ok(127.0.0.1:59618)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.425242359Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.425667419Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"2","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.425868746Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"0","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.426022908Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"1","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.42643011Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"2","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.426463116Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"0","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.426704258Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"1","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427006193Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427088423Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"[0]R flush_numbers[0..12]: [232, 237, 236, 236, 238, 240, 240, 240, 231, 226, 235, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427213379Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"[0]R generation[0..12]: [137, 141, 140, 140, 142, 143, 143, 143, 136, 133, 139, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427234796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427250375Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"[1]R flush_numbers[0..12]: [232, 237, 236, 236, 238, 240, 240, 240, 231, 226, 235, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427264323Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"[1]R generation[0..12]: [137, 141, 140, 140, 142, 143, 143, 143, 136, 133, 139, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427278121Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427292099Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"[2]R flush_numbers[0..12]: [232, 237, 236, 236, 238, 240, 240, 240, 231, 226, 235, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427305788Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"[2]R generation[0..12]: [137, 141, 140, 140, 142, 143, 143, 143, 136, 133, 139, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427319436Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427333525Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427346913Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427363571Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.42737686Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427397007Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"":"downstairs","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427410936Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427423054Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"0","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427436453Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"1","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427449581Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"client":"2","":"downstairs","session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.42746284Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427475188Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"23a1663e-8007-471f-8d3f-140fd50bfd95 is now active with session: 0108eedf-4f27-4c7a-9f95-4117e6a1af25","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427491076Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427503725Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763,"session_id":"0108eedf-4f27-4c7a-9f95-4117e6a1af25"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:56:58.427516243Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 22 len 12288 data: 16 18 17 02/30 Read block 400 len 24576 03/30 Write block 346 len 4096 data: 15 04/30 Read block 107 len 36864 05/30 Read block 336 len 40960 06/30 Read block 488 len 28672 07/30 Read block 110 len 16384 08/30 Write block 116 len 16384 data: 14 15 14 15 09/30 Write block 376 len 12288 data: 7 8 7 10/30 Write block 333 len 40960 data: 9 8 7 9 10 11 9 10 12 14 11/30 Write block 402 len 24576 data: 15 18 16 15 16 12 12/30 Read block 325 len 8192 13/30 Read block 526 len 20480 14/30 Read block 530 len 12288 15/30 Read block 592 len 4096 16/30 Write block 493 len 20480 data: 9 11 10 11 12 17/30 Read block 428 len 28672 18/30 Read block 573 len 16384 19/30 Read block 347 len 40960 20/30 Write block 15 len 28672 data: 12 16 15 16 18 15 16 21/30 Read block 161 len 40960 22/30 Read block 8 len 40960 23/30 Write block 209 len 12288 data: 12 12 15 24/30 Read block 65 len 32768 25/30 Flush 26/30 Write block 204 len 20480 data: 12 12 13 13 11 27/30 Read block 15 len 8192 28/30 Write block 537 len 40960 data: 16 15 15 13 12 12 12 13 13 15 29/30 Write block 347 len 12288 data: 13 12 9 30/30 Write block 525 len 24576 data: 5 5 6 8 9 11 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:56:58.446533414Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1763} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 241 241 241 F F F 1 020-039 144 144 144 241 241 241 F F F 2 040-059 140 140 140 236 236 236 F F F 3 060-079 140 140 140 236 236 236 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 144 144 144 241 241 241 F F F 6 120-139 143 143 143 240 240 240 F F F 7 140-159 143 143 143 240 240 240 F F F 8 160-179 136 136 136 231 231 231 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 144 144 144 241 241 241 T T T 11 220-239 142 142 142 238 238 238 F F F 12 240-259 138 138 138 233 233 233 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 134 134 134 227 227 227 F F F 15 300-319 143 143 143 240 240 240 F F F 16 320-339 144 144 144 241 241 241 F F F 17 340-359 144 144 144 241 241 241 T T T 18 360-379 144 144 144 241 241 241 F F F 19 380-399 143 143 143 240 240 240 F F F 20 400-419 144 144 144 241 241 241 F F F 21 420-439 143 143 143 240 240 240 F F F 22 440-459 143 143 143 240 240 240 F F F 23 460-479 143 143 143 240 240 240 F F F 24 480-499 144 144 144 241 241 241 F F F 25 500-519 143 143 143 240 240 240 F F F 26 520-539 138 138 138 234 234 234 T T T 27 540-559 143 143 143 240 240 240 T T T 28 560-579 143 143 143 240 240 240 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 144, Max flush: 241 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 241 241 241 F F F 1 020-039 144 144 144 241 241 241 F F F 2 040-059 140 140 140 236 236 236 F F F 3 060-079 140 140 140 236 236 236 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 144 144 144 241 241 241 F F F 6 120-139 143 143 143 240 240 240 F F F 7 140-159 143 143 143 240 240 240 F F F 8 160-179 136 136 136 231 231 231 F F F 9 180-199 133 133 133 226 226 226 F F F 10 200-219 145 145 145 242 242 242 F F F 11 220-239 142 142 142 238 238 238 F F F 12 240-259 138 138 138 233 233 233 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 134 134 134 227 227 227 F F F 15 300-319 143 143 143 240 240 240 F F F 16 320-339 144 144 144 241 241 241 F F F 17 340-359 145 145 145 242 242 242 F F F 18 360-379 144 144 144 241 241 241 F F F 19 380-399 143 143 143 240 240 240 F F F 20 400-419 144 144 144 241 241 241 F F F 21 420-439 143 143 143 240 240 240 F F F 22 440-459 143 143 143 240 240 240 F F F 23 460-479 143 143 143 240 240 240 F F F 24 480-499 144 144 144 241 241 241 F F F 25 500-519 143 143 143 240 240 240 F F F 26 520-539 145 145 145 242 242 242 F F F 27 540-559 145 145 145 242 242 242 F F F 28 560-579 143 143 143 240 240 240 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 145, Max flush: 242 Begin loop 73 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.716388321Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.760134686Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.760807648Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.760845393Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.760860822Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.76087546Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771} {"msg":"Crucible 6d33ccd3-d506-4e76-8b03-896143460ad0 has session id: f27f7846-0357-46c2-8217-057568f5c4c8","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.761373472Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"Upstairs opts: Upstairs UUID: 6d33ccd3-d506-4e76-8b03-896143460ad0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.761420686Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"Crucible stats registered with UUID: 6d33ccd3-d506-4e76-8b03-896143460ad0","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.761440274Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791142428Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791185223Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771} {"msg":"6d33ccd3-d506-4e76-8b03-896143460ad0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791220809Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791242757Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"0","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791283172Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"0","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791318448Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"1","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791354513Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"1","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.79138866Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"2","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791451972Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"2","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791714561Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791752247Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791775524Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:39392)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791799961Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:34243)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791825678Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:61246)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.791854365Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.792255648Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"0","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.792298853Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"2","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.792456015Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"1","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.792989733Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"0","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793020839Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"1","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793037467Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"2","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793571685Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793599042Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"[0]R flush_numbers[0..12]: [241, 241, 236, 236, 238, 241, 240, 240, 231, 226, 242, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793684232Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"[0]R generation[0..12]: [144, 144, 140, 140, 142, 144, 143, 143, 136, 133, 145, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793705029Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793719717Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"[1]R flush_numbers[0..12]: [241, 241, 236, 236, 238, 241, 240, 240, 231, 226, 242, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793734026Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"[1]R generation[0..12]: [144, 144, 140, 140, 142, 144, 143, 143, 136, 133, 145, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793748914Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793763232Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"[2]R flush_numbers[0..12]: [241, 241, 236, 236, 238, 241, 240, 240, 231, 226, 242, 238]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793777081Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"[2]R generation[0..12]: [144, 144, 140, 140, 142, 144, 143, 143, 136, 133, 145, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793795239Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793809967Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793823045Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793836464Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793917694Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793936652Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"":"downstairs","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793950341Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793962519Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"0","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793976187Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"1","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.793990236Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"client":"2","":"downstairs","session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.794003514Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.794016053Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"6d33ccd3-d506-4e76-8b03-896143460ad0 is now active with session: f27f7846-0357-46c2-8217-057568f5c4c8","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.794028801Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.79404093Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771,"session_id":"f27f7846-0357-46c2-8217-057568f5c4c8"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:02.794053289Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 374 len 4096 data: 6 02/30 Write block 297 len 32768 data: 9 11 12 12 13 16 14 10 03/30 Write block 353 len 12288 data: 5 8 7 04/30 Read block 33 len 8192 05/30 Read block 96 len 12288 06/30 Write block 488 len 40960 data: 13 12 9 8 9 10 12 11 12 13 07/30 Write block 188 len 24576 data: 10 13 12 14 10 12 08/30 Read block 374 len 40960 09/30 Read block 219 len 4096 10/30 Read block 596 len 12288 11/30 Read block 193 len 8192 12/30 Write block 556 len 32768 data: 7 8 9 10 9 10 12 12 13/30 Write block 210 len 4096 data: 13 14/30 Write block 202 len 12288 data: 10 11 13 15/30 Write block 46 len 20480 data: 14 11 10 11 11 16/30 Read block 242 len 16384 17/30 Read block 490 len 28672 18/30 Read block 546 len 28672 19/30 Write block 188 len 20480 data: 11 14 13 15 11 20/30 Read block 44 len 20480 21/30 Write block 239 len 16384 data: 9 8 9 10 22/30 Read block 468 len 12288 23/30 Read block 354 len 28672 24/30 Read block 36 len 36864 25/30 Flush 26/30 Write block 451 len 20480 data: 11 11 11 10 12 27/30 Read block 37 len 4096 28/30 Write block 162 len 4096 data: 20 29/30 Read block 427 len 4096 30/30 Write block 227 len 16384 data: 10 14 13 15 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:02.812344214Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1771} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 241 241 241 F F F 1 020-039 144 144 144 241 241 241 F F F 2 040-059 146 146 146 243 243 243 F F F 3 060-079 140 140 140 236 236 236 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 144 144 144 241 241 241 F F F 6 120-139 143 143 143 240 240 240 F F F 7 140-159 143 143 143 240 240 240 F F F 8 160-179 136 136 136 231 231 231 T T T 9 180-199 146 146 146 243 243 243 F F F 10 200-219 146 146 146 243 243 243 F F F 11 220-239 146 146 146 243 243 243 T T T 12 240-259 146 146 146 243 243 243 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 146 146 146 243 243 243 F F F 15 300-319 146 146 146 243 243 243 F F F 16 320-339 144 144 144 241 241 241 F F F 17 340-359 146 146 146 243 243 243 F F F 18 360-379 146 146 146 243 243 243 F F F 19 380-399 143 143 143 240 240 240 F F F 20 400-419 144 144 144 241 241 241 F F F 21 420-439 143 143 143 240 240 240 F F F 22 440-459 143 143 143 240 240 240 T T T 23 460-479 143 143 143 240 240 240 F F F 24 480-499 146 146 146 243 243 243 F F F 25 500-519 143 143 143 240 240 240 F F F 26 520-539 145 145 145 242 242 242 F F F 27 540-559 146 146 146 243 243 243 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 146, Max flush: 243 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 241 241 241 F F F 1 020-039 144 144 144 241 241 241 F F F 2 040-059 146 146 146 243 243 243 F F F 3 060-079 140 140 140 236 236 236 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 144 144 144 241 241 241 F F F 6 120-139 143 143 143 240 240 240 F F F 7 140-159 143 143 143 240 240 240 F F F 8 160-179 147 147 147 244 244 244 F F F 9 180-199 146 146 146 243 243 243 F F F 10 200-219 146 146 146 243 243 243 F F F 11 220-239 147 147 147 244 244 244 F F F 12 240-259 146 146 146 243 243 243 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 146 146 146 243 243 243 F F F 15 300-319 146 146 146 243 243 243 F F F 16 320-339 144 144 144 241 241 241 F F F 17 340-359 146 146 146 243 243 243 F F F 18 360-379 146 146 146 243 243 243 F F F 19 380-399 143 143 143 240 240 240 F F F 20 400-419 144 144 144 241 241 241 F F F 21 420-439 143 143 143 240 240 240 F F F 22 440-459 147 147 147 244 244 244 F F F 23 460-479 143 143 143 240 240 240 F F F 24 480-499 146 146 146 243 243 243 F F F 25 500-519 143 143 143 240 240 240 F F F 26 520-539 145 145 145 242 242 242 F F F 27 540-559 146 146 146 243 243 243 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 147, Max flush: 244 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.080355755Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.124245063Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.124910896Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.124948941Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.124963619Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.124978557Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779} {"msg":"Crucible 9b39b61b-e2dd-40a3-a239-eff0f4f81bab has session id: a328a4e8-6c78-4769-87af-2e5f5a7c3074","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.12539064Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Upstairs opts: Upstairs UUID: 9b39b61b-e2dd-40a3-a239-eff0f4f81bab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.125438784Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Crucible stats registered with UUID: 9b39b61b-e2dd-40a3-a239-eff0f4f81bab","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.125463571Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155155296Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155202141Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779} {"msg":"9b39b61b-e2dd-40a3-a239-eff0f4f81bab active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155273813Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155320747Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"0","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155372881Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"0","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155412117Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"1","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155450032Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"1","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155485518Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"2","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155522853Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"2","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.1555573Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155589976Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"ds_connection connected from Ok(127.0.0.1:34299)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155619922Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155653348Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"ds_connection connected from Ok(127.0.0.1:44712)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155682965Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"ds_connection connected from Ok(127.0.0.1:60848)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.155714511Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.156131782Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"1","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.156261127Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"0","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.156402751Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"2","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.156918171Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"0","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.156959606Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"1","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.156991062Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"2","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157539978Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157568325Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"[0]R flush_numbers[0..12]: [241, 241, 243, 236, 238, 241, 240, 240, 244, 243, 243, 244]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157627258Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"[0]R generation[0..12]: [144, 144, 146, 140, 142, 144, 143, 143, 147, 146, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157648696Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157663564Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"[1]R flush_numbers[0..12]: [241, 241, 243, 236, 238, 241, 240, 240, 244, 243, 243, 244]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157681921Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"[1]R generation[0..12]: [144, 144, 146, 140, 142, 144, 143, 143, 147, 146, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.15769655Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157710858Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"[2]R flush_numbers[0..12]: [241, 241, 243, 236, 238, 241, 240, 240, 244, 243, 243, 244]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157724916Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"[2]R generation[0..12]: [144, 144, 146, 140, 142, 144, 143, 143, 147, 146, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157738695Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157752683Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157765852Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.15777923Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:57:07.157792139Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157855781Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"":"downstairs","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.15787179Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157884158Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"0","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157898456Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"1","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157911655Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"client":"2","":"downstairs","session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157924963Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157939942Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"9b39b61b-e2dd-40a3-a239-eff0f4f81bab is now active with session: a328a4e8-6c78-4769-87af-2e5f5a7c3074","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.15795298Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157964988Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779,"session_id":"a328a4e8-6c78-4769-87af-2e5f5a7c3074"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:07.157977467Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 110 len 40960 data: 8 7 8 5 7 7 15 16 15 16 02/30 Read block 550 len 40960 03/30 Read block 95 len 24576 04/30 Read block 329 len 12288 05/30 Write block 541 len 12288 data: 13 13 13 06/30 Read block 226 len 32768 07/30 Read block 359 len 8192 08/30 Write block 100 len 36864 data: 9 9 10 11 11 13 10 12 13 09/30 Read block 85 len 28672 10/30 Write block 107 len 36864 data: 13 14 11 9 8 9 6 8 8 11/30 Read block 575 len 28672 12/30 Read block 184 len 16384 13/30 Read block 225 len 36864 14/30 Read block 241 len 16384 15/30 Flush 16/30 Write block 124 len 4096 data: 14 17/30 Write block 222 len 20480 data: 11 10 12 8 9 18/30 Write block 481 len 12288 data: 14 12 9 19/30 Read block 410 len 24576 20/30 Flush 21/30 Write block 521 len 28672 data: 11 10 10 9 6 6 7 22/30 Read block 426 len 20480 23/30 Read block 202 len 36864 24/30 Write block 18 len 20480 data: 17 19 16 17 17 25/30 Write block 396 len 28672 data: 6 14 12 13 12 13 16 26/30 Write block 418 len 24576 data: 11 15 15 15 13 13 27/30 Read block 148 len 28672 28/30 Read block 297 len 20480 29/30 Read block 243 len 4096 30/30 Write block 11 len 36864 data: 11 12 12 12 13 17 16 18 20 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 7 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:07.179320737Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1779} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 241 241 241 T T T 1 020-039 144 144 144 241 241 241 T T T 2 040-059 146 146 146 243 243 243 F F F 3 060-079 140 140 140 236 236 236 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 148 148 148 245 245 245 F F F 6 120-139 148 148 148 246 246 246 F F F 7 140-159 143 143 143 240 240 240 F F F 8 160-179 147 147 147 244 244 244 F F F 9 180-199 146 146 146 243 243 243 F F F 10 200-219 146 146 146 243 243 243 F F F 11 220-239 148 148 148 246 246 246 F F F 12 240-259 146 146 146 243 243 243 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 146 146 146 243 243 243 F F F 15 300-319 146 146 146 243 243 243 F F F 16 320-339 144 144 144 241 241 241 F F F 17 340-359 146 146 146 243 243 243 F F F 18 360-379 146 146 146 243 243 243 F F F 19 380-399 143 143 143 240 240 240 T T T 20 400-419 144 144 144 241 241 241 T T T 21 420-439 143 143 143 240 240 240 T T T 22 440-459 147 147 147 244 244 244 F F F 23 460-479 143 143 143 240 240 240 F F F 24 480-499 148 148 148 246 246 246 F F F 25 500-519 143 143 143 240 240 240 F F F 26 520-539 145 145 145 242 242 242 T T T 27 540-559 148 148 148 245 245 245 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 148, Max flush: 246 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 247 247 247 F F F 1 020-039 149 149 149 247 247 247 F F F 2 040-059 146 146 146 243 243 243 F F F 3 060-079 140 140 140 236 236 236 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 148 148 148 245 245 245 F F F 6 120-139 148 148 148 246 246 246 F F F 7 140-159 143 143 143 240 240 240 F F F 8 160-179 147 147 147 244 244 244 F F F 9 180-199 146 146 146 243 243 243 F F F 10 200-219 146 146 146 243 243 243 F F F 11 220-239 148 148 148 246 246 246 F F F 12 240-259 146 146 146 243 243 243 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 146 146 146 243 243 243 F F F 15 300-319 146 146 146 243 243 243 F F F 16 320-339 144 144 144 241 241 241 F F F 17 340-359 146 146 146 243 243 243 F F F 18 360-379 146 146 146 243 243 243 F F F 19 380-399 149 149 149 247 247 247 F F F 20 400-419 149 149 149 247 247 247 F F F 21 420-439 149 149 149 247 247 247 F F F 22 440-459 147 147 147 244 244 244 F F F 23 460-479 143 143 143 240 240 240 F F F 24 480-499 148 148 148 246 246 246 F F F 25 500-519 143 143 143 240 240 240 F F F 26 520-539 149 149 149 247 247 247 F F F 27 540-559 148 148 148 245 245 245 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 149, Max flush: 247 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.460186777Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.507218969Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.508105506Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.50815228Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.508168338Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.508184187Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787} {"msg":"Crucible 2eff3a9f-c89a-4c71-b1d9-e858045e478e has session id: 5826eb73-7399-4bbe-a649-52c42e35d438","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.508608057Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Upstairs opts: Upstairs UUID: 2eff3a9f-c89a-4c71-b1d9-e858045e478e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.508652222Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Crucible stats registered with UUID: 2eff3a9f-c89a-4c71-b1d9-e858045e478e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.508677099Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.540065967Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.54012636Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787} {"msg":"2eff3a9f-c89a-4c71-b1d9-e858045e478e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.540143367Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.540159896Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"0","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.540190722Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"0","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.54020864Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"1","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.540226768Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"1","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.540251525Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"2","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.540270243Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"2","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.540286771Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.540394938Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.540428804Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"ds_connection connected from Ok(127.0.0.1:52153)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.54046631Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"ds_connection connected from Ok(127.0.0.1:52361)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.540496796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"ds_connection connected from Ok(127.0.0.1:44552)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.540525593Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.541279515Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"2","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.54132134Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"0","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.541485411Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"1","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.541978503Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"2","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.54200766Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"0","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542099679Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"1","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542572134Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542615069Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"[0]R flush_numbers[0..12]: [247, 247, 243, 236, 238, 245, 246, 240, 244, 243, 243, 246]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542634297Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"[0]R generation[0..12]: [149, 149, 146, 140, 142, 148, 148, 143, 147, 146, 146, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542649995Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542665233Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"[1]R flush_numbers[0..12]: [247, 247, 243, 236, 238, 245, 246, 240, 244, 243, 243, 246]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542679591Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"[1]R generation[0..12]: [149, 149, 146, 140, 142, 148, 148, 143, 147, 146, 146, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.54269346Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542707528Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"[2]R flush_numbers[0..12]: [247, 247, 243, 236, 238, 245, 246, 240, 244, 243, 243, 246]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542721417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"[2]R generation[0..12]: [149, 149, 146, 140, 142, 148, 148, 143, 147, 146, 146, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542735145Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542749083Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542762112Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.54277536Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542788509Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542801747Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"":"downstairs","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:57:11.542818205Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542844492Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"0","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542870329Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"1","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542890217Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"client":"2","":"downstairs","session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542914744Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542938041Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"2eff3a9f-c89a-4c71-b1d9-e858045e478e is now active with session: 5826eb73-7399-4bbe-a649-52c42e35d438","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542960329Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:11.542974287Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787,"session_id":"5826eb73-7399-4bbe-a649-52c42e35d438"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-13T15:57:11.543011293Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 133 len 32768 data: 8 10 13 12 11 10 9 13 02/30 Read block 497 len 28672 03/30 Read block 586 len 8192 04/30 Flush 05/30 Write block 6 len 20480 data: 8 12 11 9 11 06/30 Flush 07/30 Write block 496 len 20480 data: 13 14 13 14 16 08/30 Write block 401 len 8192 data: 14 17 09/30 Read block 558 len 28672 10/30 Write block 114 len 16384 data: 9 9 16 17 11/30 Read block 237 len 8192 12/30 Write block 140 len 36864 data: 14 13 11 11 13 11 9 10 14 13/30 Write block 395 len 8192 data: 6 7 14/30 Read block 321 len 20480 15/30 Write block 423 len 32768 data: 14 13 11 11 9 10 10 10 16/30 Read block 393 len 20480 17/30 Flush 18/30 Write block 70 len 32768 data: 13 11 9 9 10 8 4 5 19/30 Read block 556 len 40960 20/30 Read block 18 len 32768 21/30 Read block 459 len 36864 22/30 Write block 540 len 28672 data: 14 14 14 14 14 14 16 23/30 Write block 291 len 28672 data: 14 15 13 13 9 7 10 24/30 Flush 25/30 Write block 387 len 32768 data: 6 6 11 9 9 7 6 7 26/30 Write block 354 len 32768 data: 9 8 7 8 10 13 11 10 27/30 Write block 54 len 16384 data: 15 15 11 11 28/30 Write block 505 len 32768 data: 11 10 9 10 9 8 8 8 29/30 Write block 253 len 20480 data: 10 11 12 10 12 30/30 Write block 500 len 40960 data: 17 16 16 16 14 12 11 10 11 10 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:11.563040546Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1787} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 249 249 249 F F F 1 020-039 149 149 149 247 247 247 F F F 2 040-059 146 146 146 243 243 243 T T T 3 060-079 150 150 150 251 251 251 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 150 150 150 250 250 250 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 150 150 150 250 250 250 F F F 8 160-179 147 147 147 244 244 244 F F F 9 180-199 146 146 146 243 243 243 F F F 10 200-219 146 146 146 243 243 243 F F F 11 220-239 148 148 148 246 246 246 F F F 12 240-259 146 146 146 243 243 243 T T T 13 260-279 139 139 139 235 235 235 F F F 14 280-299 150 150 150 251 251 251 F F F 15 300-319 146 146 146 243 243 243 F F F 16 320-339 144 144 144 241 241 241 F F F 17 340-359 146 146 146 243 243 243 T T T 18 360-379 146 146 146 243 243 243 T T T 19 380-399 150 150 150 250 250 250 T T T 20 400-419 150 150 150 250 250 250 F F F 21 420-439 150 150 150 250 250 250 F F F 22 440-459 147 147 147 244 244 244 F F F 23 460-479 143 143 143 240 240 240 F F F 24 480-499 150 150 150 250 250 250 F F F 25 500-519 150 150 150 250 250 250 T T T 26 520-539 149 149 149 247 247 247 F F F 27 540-559 150 150 150 251 251 251 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 150, Max flush: 251 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 249 249 249 F F F 1 020-039 149 149 149 247 247 247 F F F 2 040-059 151 151 151 252 252 252 F F F 3 060-079 150 150 150 251 251 251 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 150 150 150 250 250 250 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 150 150 150 250 250 250 F F F 8 160-179 147 147 147 244 244 244 F F F 9 180-199 146 146 146 243 243 243 F F F 10 200-219 146 146 146 243 243 243 F F F 11 220-239 148 148 148 246 246 246 F F F 12 240-259 151 151 151 252 252 252 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 150 150 150 251 251 251 F F F 15 300-319 146 146 146 243 243 243 F F F 16 320-339 144 144 144 241 241 241 F F F 17 340-359 151 151 151 252 252 252 F F F 18 360-379 151 151 151 252 252 252 F F F 19 380-399 151 151 151 252 252 252 F F F 20 400-419 150 150 150 250 250 250 F F F 21 420-439 150 150 150 250 250 250 F F F 22 440-459 147 147 147 244 244 244 F F F 23 460-479 143 143 143 240 240 240 F F F 24 480-499 150 150 150 250 250 250 F F F 25 500-519 151 151 151 252 252 252 F F F 26 520-539 149 149 149 247 247 247 F F F 27 540-559 150 150 150 251 251 251 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 151, Max flush: 252 Begin loop 76 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.854225962Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:15.89811987Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.898564178Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.929468062Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.93016528Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.930208335Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.930225373Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.930243171Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795} {"msg":"Crucible a0064e9b-58db-44e7-acab-17b3049f832e has session id: bc3195ff-60bf-4ff5-9949-0f9863f7dc7e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.930747493Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"Upstairs opts: Upstairs UUID: a0064e9b-58db-44e7-acab-17b3049f832e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.930781158Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"Crucible stats registered with UUID: a0064e9b-58db-44e7-acab-17b3049f832e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.930796497Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.96127503Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.961339853Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795} {"msg":"a0064e9b-58db-44e7-acab-17b3049f832e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.96136824Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.961386877Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"0","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.961431972Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"0","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.961466398Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"1","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.961548958Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"1","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.961576605Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"2","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.961601202Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"2","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.961624219Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.961649117Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.961674304Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40431)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.961974309Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60484)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.961996706Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61260)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.962022133Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.962629852Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"1","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.962669617Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"2","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.962687955Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"0","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.963383984Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"1","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.963480503Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"2","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.96350716Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"0","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964010501Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964048117Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"[0]R flush_numbers[0..12]: [249, 247, 252, 251, 238, 250, 248, 250, 244, 243, 243, 246]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964123918Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"[0]R generation[0..12]: [150, 149, 151, 150, 142, 150, 150, 150, 147, 146, 146, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964144945Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964159574Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"[1]R flush_numbers[0..12]: [249, 247, 252, 251, 238, 250, 248, 250, 244, 243, 243, 246]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964173532Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"[1]R generation[0..12]: [150, 149, 151, 150, 142, 150, 150, 150, 147, 146, 146, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.96418736Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964201489Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"[2]R flush_numbers[0..12]: [249, 247, 252, 251, 238, 250, 248, 250, 244, 243, 243, 246]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964215507Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"[2]R generation[0..12]: [150, 149, 151, 150, 142, 150, 150, 150, 147, 146, 146, 148]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964232735Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964248003Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964261312Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.9642748Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964288049Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964301207Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"":"downstairs","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964314416Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964326484Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"0","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"Wait for a query_work_queue command to finish before sending IO msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964347122Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"1","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.96436051Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"client":"2","":"downstairs","session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964373849Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964385997Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"a0064e9b-58db-44e7-acab-17b3049f832e is now active with session: bc3195ff-60bf-4ff5-9949-0f9863f7dc7e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964398676Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964410714Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795,"session_id":"bc3195ff-60bf-4ff5-9949-0f9863f7dc7e"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:15.964422993Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 64 len 4096 data: 9 02/30 Read block 73 len 4096 03/30 Read block 106 len 36864 04/30 Flush 05/30 Write block 216 len 32768 data: 13 12 14 12 12 12 12 11 06/30 Read block 405 len 4096 07/30 Read block 501 len 20480 08/30 Write block 14 len 8192 data: 13 14 09/30 Write block 457 len 28672 data: 12 13 16 15 15 16 15 10/30 Read block 230 len 12288 11/30 Read block 593 len 20480 12/30 Read block 518 len 4096 13/30 Write block 426 len 32768 data: 12 10 11 11 11 8 11 10 14/30 Read block 47 len 36864 15/30 Read block 243 len 28672 16/30 Flush 17/30 Write block 347 len 32768 data: 14 13 10 11 10 9 6 10 18/30 Write block 469 len 8192 data: 7 5 19/30 Write block 181 len 16384 data: 12 11 11 11 20/30 Write block 200 len 12288 data: 13 11 11 21/30 Write block 368 len 16384 data: 8 6 6 7 22/30 Read block 496 len 40960 23/30 Read block 285 len 16384 24/30 Read block 73 len 12288 25/30 Write block 153 len 28672 data: 14 13 13 12 12 13 13 26/30 Read block 64 len 12288 27/30 Read block 87 len 24576 28/30 Write block 143 len 28672 data: 12 14 12 10 11 15 14 29/30 Write block 354 len 40960 data: 11 9 8 9 11 14 12 11 11 10 30/30 Write block 230 len 8192 data: 16 16 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 8 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 7 Sent Done Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 3 8 Done 11 12 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:15.98355905Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1795} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 254 254 254 F F F 1 020-039 149 149 149 247 247 247 F F F 2 040-059 151 151 151 252 252 252 F F F 3 060-079 152 152 152 253 253 253 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 150 150 150 250 250 250 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 150 150 150 250 250 250 T T T 8 160-179 147 147 147 244 244 244 F F F 9 180-199 146 146 146 243 243 243 T T T 10 200-219 152 152 152 254 254 254 T T T 11 220-239 152 152 152 254 254 254 T T T 12 240-259 151 151 151 252 252 252 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 150 150 150 251 251 251 F F F 15 300-319 146 146 146 243 243 243 F F F 16 320-339 144 144 144 241 241 241 F F F 17 340-359 151 151 151 252 252 252 T T T 18 360-379 151 151 151 252 252 252 T T T 19 380-399 151 151 151 252 252 252 F F F 20 400-419 150 150 150 250 250 250 F F F 21 420-439 152 152 152 254 254 254 F F F 22 440-459 152 152 152 254 254 254 F F F 23 460-479 152 152 152 254 254 254 T T T 24 480-499 150 150 150 250 250 250 F F F 25 500-519 151 151 151 252 252 252 F F F 26 520-539 149 149 149 247 247 247 F F F 27 540-559 150 150 150 251 251 251 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 152, Max flush: 254 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 254 254 254 F F F 1 020-039 149 149 149 247 247 247 F F F 2 040-059 151 151 151 252 252 252 F F F 3 060-079 152 152 152 253 253 253 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 150 150 150 250 250 250 F F F 6 120-139 150 150 150 248 248 248 F F F 7 140-159 153 153 153 255 255 255 F F F 8 160-179 147 147 147 244 244 244 F F F 9 180-199 153 153 153 255 255 255 F F F 10 200-219 153 153 153 255 255 255 F F F 11 220-239 153 153 153 255 255 255 F F F 12 240-259 151 151 151 252 252 252 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 150 150 150 251 251 251 F F F 15 300-319 146 146 146 243 243 243 F F F 16 320-339 144 144 144 241 241 241 F F F 17 340-359 153 153 153 255 255 255 F F F 18 360-379 153 153 153 255 255 255 F F F 19 380-399 151 151 151 252 252 252 F F F 20 400-419 150 150 150 250 250 250 F F F 21 420-439 152 152 152 254 254 254 F F F 22 440-459 152 152 152 254 254 254 F F F 23 460-479 153 153 153 255 255 255 F F F 24 480-499 150 150 150 250 250 250 F F F 25 500-519 151 151 151 252 252 252 F F F 26 520-539 149 149 149 247 247 247 F F F 27 540-559 150 150 150 251 251 251 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 153, Max flush: 255 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.315366387Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.359803512Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.360529367Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.360567042Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.360581961Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.360600569Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803} {"msg":"Crucible f13fff20-06e6-4900-881f-a3eb181e552f has session id: f98b8240-4b74-4aa8-b598-eec2448ab017","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.361110059Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Upstairs opts: Upstairs UUID: f13fff20-06e6-4900-881f-a3eb181e552f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.361162593Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Crucible stats registered with UUID: f13fff20-06e6-4900-881f-a3eb181e552f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.36118797Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391465567Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391535219Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803} {"msg":"f13fff20-06e6-4900-881f-a3eb181e552f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391567265Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391597112Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"0","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391647406Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"0","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391685102Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"1","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391721247Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"1","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391743185Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"2","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391762002Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"2","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.39178121Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391819956Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391854952Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"ds_connection connected from Ok(127.0.0.1:56171)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391884368Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"ds_connection connected from Ok(127.0.0.1:55459)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.391922934Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"ds_connection connected from Ok(127.0.0.1:42821)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.392234397Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.392661248Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"2","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.392711432Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"0","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.392976611Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"1","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.393480082Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"2","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.393558573Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"1","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.393599208Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"0","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394069263Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394136225Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"[0]R flush_numbers[0..12]: [254, 247, 252, 253, 238, 250, 248, 255, 244, 255, 255, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394205577Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"[0]R generation[0..12]: [152, 149, 151, 152, 142, 150, 150, 153, 147, 153, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394225695Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394244933Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"[1]R flush_numbers[0..12]: [254, 247, 252, 253, 238, 250, 248, 255, 244, 255, 255, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394259451Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"[1]R generation[0..12]: [152, 149, 151, 152, 142, 150, 150, 153, 147, 153, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394273229Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394287238Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"[2]R flush_numbers[0..12]: [254, 247, 252, 253, 238, 250, 248, 255, 244, 255, 255, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394300986Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"[2]R generation[0..12]: [152, 149, 151, 152, 142, 150, 150, 153, 147, 153, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394314785Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394328903Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394342241Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.39435584Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Next flush: 256","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394368948Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394382127Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"":"downstairs","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394395365Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394407514Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803Wait for a query_work_queue command to finish before sending IO ,"client":"0","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394430341Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"1","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394445889Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"client":"2","":"downstairs","session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394459398Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394471766Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"f13fff20-06e6-4900-881f-a3eb181e552f is now active with session: f98b8240-4b74-4aa8-b598-eec2448ab017","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394484605Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394496703Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803,"session_id":"f98b8240-4b74-4aa8-b598-eec2448ab017"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:20.394508842Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 209 len 40960 data: 13 14 16 16 15 14 15 14 13 15 02/30 Flush 03/30 Write block 434 len 4096 data: 9 04/30 Write block 200 len 40960 data: 14 12 12 12 14 13 14 14 12 14 05/30 Write block 67 len 8192 data: 16 15 06/30 Write block 361 len 36864 data: 12 12 11 10 11 11 10 9 7 07/30 Read block 304 len 28672 08/30 Write block 298 len 36864 data: 12 13 13 14 17 15 11 10 9 09/30 Write block 313 len 4096 data: 8 10/30 Read block 342 len 16384 11/30 Write block 109 len 24576 data: 12 10 9 10 7 10 12/30 Write block 335 len 32768 data: 8 10 11 12 10 11 13 15 13/30 Write block 34 len 4096 data: 4 14/30 Flush 15/30 Write block 323 len 4096 data: 12 16/30 Read block 560 len 16384 17/30 Write block 258 len 24576 data: 13 14 14 14 14 15 18/30 Write block 11 len 20480 data: 12 13 13 14 15 19/30 Write block 368 len 28672 data: 10 8 7 8 7 8 7 20/30 Write block 262 len 4096 data: 15 21/30 Read block 503 len 40960 22/30 Write block 532 len 40960 data: 15 14 14 12 14 17 16 16 15 15 23/30 Write block 378 len 12288 data: 8 8 9 24/30 Read block 455 len 4096 25/30 Flush 26/30 Write block 49 len 24576 data: 12 12 10 13 12 16 27/30 Write block 128 len 24576 data: 15 11 7 8 8 9 28/30 Write block 262 len 40960 data: 16 16 13 13 15 14 10 9 9 8 29/30 Read block 551 len 36864 30/30 Read block 191 len 40960 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 4 5 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:20.414202274Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1803} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 258 258 258 F F F 1 020-039 154 154 154 257 257 257 F F F 2 040-059 151 151 151 252 252 252 T T T 3 060-079 154 154 154 257 257 257 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 154 154 154 257 257 257 F F F 6 120-139 150 150 150 248 248 248 T T T 7 140-159 153 153 153 255 255 255 F F F 8 160-179 147 147 147 244 244 244 F F F 9 180-199 153 153 153 255 255 255 F F F 10 200-219 154 154 154 257 257 257 F F F 11 220-239 153 153 153 255 255 255 F F F 12 240-259 154 154 154 258 258 258 F F F 13 260-279 154 154 154 258 258 258 T T T 14 280-299 154 154 154 257 257 257 F F F 15 300-319 154 154 154 257 257 257 F F F 16 320-339 154 154 154 258 258 258 F F F 17 340-359 154 154 154 257 257 257 F F F 18 360-379 154 154 154 258 258 258 F F F 19 380-399 154 154 154 258 258 258 F F F 20 400-419 150 150 150 250 250 250 F F F 21 420-439 154 154 154 257 257 257 F F F 22 440-459 152 152 152 254 254 254 F F F 23 460-479 153 153 153 255 255 255 F F F 24 480-499 150 150 150 250 250 250 F F F 25 500-519 151 151 151 252 252 252 F F F 26 520-539 154 154 154 258 258 258 F F F 27 540-559 154 154 154 258 258 258 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 154, Max flush: 258 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 258 258 258 F F F 1 020-039 154 154 154 257 257 257 F F F 2 040-059 155 155 155 259 259 259 F F F 3 060-079 154 154 154 257 257 257 F F F 4 080-099 142 142 142 238 238 238 F F F 5 100-119 154 154 154 257 257 257 F F F 6 120-139 155 155 155 259 259 259 F F F 7 140-159 153 153 153 255 255 255 F F F 8 160-179 147 147 147 244 244 244 F F F 9 180-199 153 153 153 255 255 255 F F F 10 200-219 154 154 154 257 257 257 F F F 11 220-239 153 153 153 255 255 255 F F F 12 240-259 154 154 154 258 258 258 F F F 13 260-279 155 155 155 259 259 259 F F F 14 280-299 154 154 154 257 257 257 F F F 15 300-319 154 154 154 257 257 257 F F F 16 320-339 154 154 154 258 258 258 F F F 17 340-359 154 154 154 257 257 257 F F F 18 360-379 154 154 154 258 258 258 F F F 19 380-399 154 154 154 258 258 258 F F F 20 400-419 150 150 150 250 250 250 F F F 21 420-439 154 154 154 257 257 257 F F F 22 440-459 152 152 152 254 254 254 F F F 23 460-479 153 153 153 255 255 255 F F F 24 480-499 150 150 150 250 250 250 F F F 25 500-519 151 151 151 252 252 252 F F F 26 520-539 154 154 154 258 258 258 F F F 27 540-559 154 154 154 258 258 258 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 139 139 139 235 235 235 F F F Max gen: 155, Max flush: 259 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.686442011Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.730644033Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.731389046Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.731426362Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.731440571Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.731454749Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811} {"msg":"Crucible 0123d470-ba8a-4734-bc10-f26d2968cdf5 has session id: 20792888-19c9-43ac-88aa-c6f8ef58b791","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.73187796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Upstairs opts: Upstairs UUID: 0123d470-ba8a-4734-bc10-f26d2968cdf5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.731924324Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Crucible stats registered with UUID: 0123d470-ba8a-4734-bc10-f26d2968cdf5","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.731944562Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.762676356Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.762716521Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811} {"msg":"0123d470-ba8a-4734-bc10-f26d2968cdf5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.762737839Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.762754157Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"0","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.762783163Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"0","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.762800911Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"1","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.762826508Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"1","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.762924627Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"2","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.762970881Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"2","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.763009677Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.763044663Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.763080609Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"ds_connection connected from Ok(127.0.0.1:53017)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.763112795Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"ds_connection connected from Ok(127.0.0.1:49245)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.763181997Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"ds_connection connected from Ok(127.0.0.1:37389)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.763219232Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.763807144Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"0","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.763847599Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"1","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.764049355Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"2","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.764634947Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"1","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.764665194Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"0","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.764684581Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"2","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765155226Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765269613Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"[0]R flush_numbers[0..12]: [258, 257, 259, 257, 238, 257, 259, 255, 244, 255, 257, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765338395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"[0]R generation[0..12]: [154, 154, 155, 154, 142, 154, 155, 153, 147, 153, 154, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765359193Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765374161Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"[1]R flush_numbers[0..12]: [258, 257, 259, 257, 238, 257, 259, 255, 244, 255, 257, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765388559Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"[1]R generation[0..12]: [154, 154, 155, 154, 142, 154, 155, 153, 147, 153, 154, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765402398Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765416356Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"[2]R flush_numbers[0..12]: [258, 257, 259, 257, 238, 257, 259, 255, 244, 255, 257, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765430294Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"[2]R generation[0..12]: [154, 154, 155, 154, 142, 154, 155, 153, 147, 153, 154, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765444113Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765458081Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.76547113Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765484678Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Next flush: 260","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765501106Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765514664Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"":"downstairs","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765528363Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765540441Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"0","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:57:24.76555395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"1","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765574478Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"client":"2","":"downstairs","session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765594535Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765609413Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"0123d470-ba8a-4734-bc10-f26d2968cdf5 is now active with session: 20792888-19c9-43ac-88aa-c6f8ef58b791","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765622282Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765634331Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811,"session_id":"20792888-19c9-43ac-88aa-c6f8ef58b791"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:24.765646549Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 117 len 8192 data: 18 16 02/30 Read block 56 len 12288 03/30 Write block 172 len 36864 data: 10 7 7 6 8 10 10 10 12 04/30 Read block 301 len 28672 05/30 Write block 411 len 36864 data: 17 18 18 15 15 13 13 12 16 06/30 Read block 404 len 28672 07/30 Read block 138 len 16384 08/30 Write block 535 len 32768 data: 13 15 18 17 17 16 16 15 09/30 Write block 89 len 32768 data: 19 16 18 16 18 14 14 15 10/30 Write block 242 len 8192 data: 11 7 11/30 Read block 118 len 32768 12/30 Read block 27 len 20480 13/30 Read block 586 len 12288 14/30 Write block 448 len 12288 data: 7 11 10 15/30 Read block 139 len 40960 16/30 Flush 17/30 Write block 373 len 8192 data: 9 8 18/30 Read block 121 len 36864 19/30 Read block 9 len 32768 20/30 Write block 598 len 4096 data: 7 21/30 Read block 186 len 32768 22/30 Read block 425 len 4096 23/30 Read block 462 len 8192 24/30 Write block 288 len 4096 data: 9 25/30 Write block 183 len 12288 data: 12 12 10 26/30 Read block 468 len 32768 27/30 Write block 58 len 36864 data: 10 11 8 8 8 9 10 9 13 28/30 Write block 438 len 16384 data: 12 10 8 7 29/30 Read block 114 len 28672 30/30 Write block 248 len 36864 data: 9 7 8 7 8 11 12 13 11 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 2 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:24.785910264Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1811} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 258 258 258 F F F 1 020-039 154 154 154 257 257 257 F F F 2 040-059 155 155 155 259 259 259 T T T 3 060-079 154 154 154 257 257 257 T T T 4 080-099 156 156 156 260 260 260 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 155 155 155 259 259 259 F F F 7 140-159 153 153 153 255 255 255 F F F 8 160-179 156 156 156 260 260 260 F F F 9 180-199 156 156 156 260 260 260 T T T 10 200-219 154 154 154 257 257 257 F F F 11 220-239 153 153 153 255 255 255 F F F 12 240-259 156 156 156 260 260 260 T T T 13 260-279 155 155 155 259 259 259 F F F 14 280-299 154 154 154 257 257 257 T T T 15 300-319 154 154 154 257 257 257 F F F 16 320-339 154 154 154 258 258 258 F F F 17 340-359 154 154 154 257 257 257 F F F 18 360-379 154 154 154 258 258 258 T T T 19 380-399 154 154 154 258 258 258 F F F 20 400-419 156 156 156 260 260 260 F F F 21 420-439 154 154 154 257 257 257 T T T 22 440-459 156 156 156 260 260 260 T T T 23 460-479 153 153 153 255 255 255 F F F 24 480-499 150 150 150 250 250 250 F F F 25 500-519 151 151 151 252 252 252 F F F 26 520-539 156 156 156 260 260 260 F F F 27 540-559 156 156 156 260 260 260 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 139 139 139 235 235 235 T T T Max gen: 156, Max flush: 260 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 258 258 258 F F F 1 020-039 154 154 154 257 257 257 F F F 2 040-059 157 157 157 261 261 261 F F F 3 060-079 157 157 157 261 261 261 F F F 4 080-099 156 156 156 260 260 260 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 155 155 155 259 259 259 F F F 7 140-159 153 153 153 255 255 255 F F F 8 160-179 156 156 156 260 260 260 F F F 9 180-199 157 157 157 261 261 261 F F F 10 200-219 154 154 154 257 257 257 F F F 11 220-239 153 153 153 255 255 255 F F F 12 240-259 157 157 157 261 261 261 F F F 13 260-279 155 155 155 259 259 259 F F F 14 280-299 157 157 157 261 261 261 F F F 15 300-319 154 154 154 257 257 257 F F F 16 320-339 154 154 154 258 258 258 F F F 17 340-359 154 154 154 257 257 257 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 154 154 154 258 258 258 F F F 20 400-419 156 156 156 260 260 260 F F F 21 420-439 157 157 157 261 261 261 F F F 22 440-459 157 157 157 261 261 261 F F F 23 460-479 153 153 153 255 255 255 F F F 24 480-499 150 150 150 250 250 250 F F F 25 500-519 151 151 151 252 252 252 F F F 26 520-539 156 156 156 260 260 260 F F F 27 540-559 156 156 156 260 260 260 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 157 157 157 261 261 261 F F F Max gen: 157, Max flush: 261 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.082242411Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.126525134Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.127436118Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.127482612Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.12750126Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.127519248Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819} {"msg":"Crucible 2f05ac4d-605b-4aa1-a71e-e0f2cf458157 has session id: d2acc821-5cec-4b99-9f4d-75e5830d61fa","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.12801571Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"Upstairs opts: Upstairs UUID: 2f05ac4d-605b-4aa1-a71e-e0f2cf458157, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.128062454Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"Crucible stats registered with UUID: 2f05ac4d-605b-4aa1-a71e-e0f2cf458157","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.128087602Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158190989Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158249442Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819} {"msg":"2f05ac4d-605b-4aa1-a71e-e0f2cf458157 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158272909Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158291807Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"0","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158337752Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"0","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158374048Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"1","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158411293Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"1","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158444079Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"2","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158475696Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"2","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158502533Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158529789Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158564015Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:47870)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158605661Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:55493)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158643816Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:57075)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.158681932Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.159531982Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"0","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.159573697Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"2","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.159594845Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"1","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.160391532Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"1","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.160432667Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"0","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.160451605Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"2","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.160975584Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.16101205Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"[0]R flush_numbers[0..12]: [258, 257, 261, 261, 260, 260, 259, 255, 260, 261, 257, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161128706Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"[0]R generation[0..12]: [154, 154, 157, 157, 156, 156, 155, 153, 156, 157, 154, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161158533Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161174061Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"[1]R flush_numbers[0..12]: [258, 257, 261, 261, 260, 260, 259, 255, 260, 261, 257, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161188679Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"[1]R generation[0..12]: [154, 154, 157, 157, 156, 156, 155, 153, 156, 157, 154, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161202927Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161217306Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"[2]R flush_numbers[0..12]: [258, 257, 261, 261, 260, 260, 259, 255, 260, 261, 257, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161234904Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"[2]R generation[0..12]: [154, 154, 157, 157, 156, 156, 155, 153, 156, 157, 154, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161270279Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161291597Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161311205Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161337732Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161357009Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161377087Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"":"downstairs","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161396495Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161414123Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"0","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.1614344Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"1","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161454348Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"client":"2","":"downstairs","session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161474366Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161493344Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"2f05ac4d-605b-4aa1-a71e-e0f2cf458157 is now active with session: d2acc821-5cec-4b99-9f4d-75e5830d61fa","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161513401Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161530949Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819,"session_id":"d2acc821-5cec-4b99-9f4d-75e5830d61fa"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:29.161555906Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 311 len 16384 data: 6 6 9 9 02/30 Read block 549 len 12288 03/30 Write block 420 len 32768 data: 16 16 14 15 14 12 13 11 04/30 Flush 05/30 Write block 448 len 40960 data: 8 12 11 12 12 12 11 13 12 13 06/30 Read block 553 len 40960 07/30 Write block 410 len 32768 data: 18 18 19 19 16 16 14 14 08/30 Read block 272 len 20480 09/30 Write block 73 len 12288 data: 10 11 9 10/30 Write block 538 len 28672 data: 18 18 17 17 16 15 15 11/30 Write block 2 len 12288 data: 4 5 6 12/30 Write block 226 len 20480 data: 10 11 15 14 17 13/30 Read block 500 len 8192 14/30 Read block 459 len 28672 15/30 Flush 16/30 Write block 396 len 20480 data: 8 15 13 14 13 17/30 Write block 594 len 16384 data: 16 13 12 8 18/30 Flush 19/30 Write block 508 len 36864 data: 12 11 9 9 9 7 9 10 13 20/30 Read block 371 len 12288 21/30 Read block 483 len 20480 22/30 Read block 228 len 16384 23/30 Write block 16 len 32768 data: 18 17 19 21 17 18 18 19 24/30 Read block 336 len 28672 25/30 Read block 82 len 40960 26/30 Write block 200 len 32768 data: 15 13 13 13 15 14 15 15 27/30 Write block 326 len 28672 data: 13 13 15 12 11 9 10 28/30 Write block 63 len 36864 data: 10 11 10 14 17 16 12 14 12 29/30 Write block 159 len 24576 data: 14 16 17 21 18 17 30/30 Read block 419 len 20480 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:29.183012303Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1819} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 263 263 263 T T T 1 020-039 154 154 154 257 257 257 T T T 2 040-059 157 157 157 261 261 261 F F F 3 060-079 158 158 158 263 263 263 T T T 4 080-099 156 156 156 260 260 260 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 155 155 155 259 259 259 F F F 7 140-159 153 153 153 255 255 255 T T T 8 160-179 156 156 156 260 260 260 T T T 9 180-199 157 157 157 261 261 261 F F F 10 200-219 154 154 154 257 257 257 T T T 11 220-239 158 158 158 263 263 263 F F F 12 240-259 157 157 157 261 261 261 F F F 13 260-279 155 155 155 259 259 259 F F F 14 280-299 157 157 157 261 261 261 F F F 15 300-319 158 158 158 262 262 262 F F F 16 320-339 154 154 154 258 258 258 T T T 17 340-359 154 154 154 257 257 257 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 158 158 158 264 264 264 F F F 20 400-419 158 158 158 264 264 264 F F F 21 420-439 158 158 158 262 262 262 F F F 22 440-459 158 158 158 263 263 263 F F F 23 460-479 153 153 153 255 255 255 F F F 24 480-499 150 150 150 250 250 250 F F F 25 500-519 151 151 151 252 252 252 T T T 26 520-539 158 158 158 263 263 263 F F F 27 540-559 158 158 158 263 263 263 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 158 158 158 264 264 264 F F F Max gen: 158, Max flush: 264 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 265 265 265 F F F 1 020-039 159 159 159 265 265 265 F F F 2 040-059 157 157 157 261 261 261 F F F 3 060-079 159 159 159 265 265 265 F F F 4 080-099 156 156 156 260 260 260 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 155 155 155 259 259 259 F F F 7 140-159 159 159 159 265 265 265 F F F 8 160-179 159 159 159 265 265 265 F F F 9 180-199 157 157 157 261 261 261 F F F 10 200-219 159 159 159 265 265 265 F F F 11 220-239 158 158 158 263 263 263 F F F 12 240-259 157 157 157 261 261 261 F F F 13 260-279 155 155 155 259 259 259 F F F 14 280-299 157 157 157 261 261 261 F F F 15 300-319 158 158 158 262 262 262 F F F 16 320-339 159 159 159 265 265 265 F F F 17 340-359 154 154 154 257 257 257 F F F 18 360-379 157 157 157 261 261 261 F F F 19 380-399 158 158 158 264 264 264 F F F 20 400-419 158 158 158 264 264 264 F F F 21 420-439 158 158 158 262 262 262 F F F 22 440-459 158 158 158 263 263 263 F F F 23 460-479 153 153 153 255 255 255 F F F 24 480-499 150 150 150 250 250 250 F F F 25 500-519 159 159 159 265 265 265 F F F 26 520-539 158 158 158 263 263 263 F F F 27 540-559 158 158 158 263 263 263 F F F 28 560-579 146 146 146 243 243 243 F F F 29 580-599 158 158 158 264 264 264 F F F Max gen: 159, Max flush: 265 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.485598449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.530250669Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.531188519Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.531230764Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.531250002Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.53126468Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827} {"msg":"Crucible 61d22f72-79dc-418e-a2ac-ea68aba9c506 has session id: a4a55f8b-9644-4921-a733-51023d72c6af","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.531851312Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"Upstairs opts: Upstairs UUID: 61d22f72-79dc-418e-a2ac-ea68aba9c506, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.531897897Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"Crucible stats registered with UUID: 61d22f72-79dc-418e-a2ac-ea68aba9c506","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.531917074Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.562716941Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.562757116Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827} {"msg":"61d22f72-79dc-418e-a2ac-ea68aba9c506 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.562792172Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.56281031Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"0","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.562840806Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"0","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.562858644Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"1","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.562886281Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"1","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.562918047Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"2","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.562953163Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"2","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.562994688Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.563029924Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.56306403Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"ds_connection connected from Ok(127.0.0.1:47958)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.563099566Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"ds_connection connected from Ok(127.0.0.1:60578)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.56332551Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"ds_connection connected from Ok(127.0.0.1:64111)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.563373624Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.563927029Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"2","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.563957956Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"0","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.564088741Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"1","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.564635897Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"0","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.564668453Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"2","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.564830524Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"1","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565165115Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565240496Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"[0]R flush_numbers[0..12]: [265, 265, 261, 265, 260, 260, 259, 265, 265, 261, 265, 263]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565419615Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairs","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"[0]R generation[0..12]: [159, 159, 157, 159, 156, 156, 155, 159, 159, 157, 159, 158]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565447242Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairs","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.56546309Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairs","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"[1]R flush_numbers[0..12]: [265, 265, 261, 265, 260, 260, 259, 265, 265, 261, 265, 263]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565478528Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairs","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"[1]R generation[0..12]: [159, 159, 157, 159, 156, 156, 155, 159, 159, 157, 159, 158]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565492637Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairs","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565506755Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairs","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"[2]R flush_numbers[0..12]: [265, 265, 261, 265, 260, 260, 259, 265, 265, 261, 265, 263]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565520793Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairs","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"[2]R generation[0..12]: [159, 159, 157, 159, 156, 156, 155, 159, 159, 157, 159, 158]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565534562Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairs","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.56554853Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairs","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565561509Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairs","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565574887Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairs","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565587766Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairsWait for a query_work_queue command to finish before sending IO ","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565607513Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"":"downstairs","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565621022Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.56563327Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"0","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565649278Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"1","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565662707Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"client":"2","":"downstairs","session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565675985Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msgDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565692133Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"61d22f72-79dc-418e-a2ac-ea68aba9c506 is now active with session: a4a55f8b-9644-4921-a733-51023d72c6af","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565705092Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.5657172Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827,"session_id":"a4a55f8b-9644-4921-a733-51023d72c6af"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:33.565729539Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 155 len 8192 data: 14 13 02/30 Read block 352 len 12288 03/30 Read block 327 len 28672 04/30 Flush 05/30 Write block 258 len 20480 data: 14 15 15 15 17 06/30 Flush 07/30 Write block 325 len 28672 data: 14 14 14 16 13 12 10 08/30 Flush 09/30 Write block 576 len 4096 data: 13 10/30 Write block 362 len 16384 data: 13 12 11 12 11/30 Write block 469 len 8192 data: 8 6 12/30 Flush 13/30 Flush 14/30 Write block 250 len 36864 data: 9 8 9 12 13 14 12 13 15 15/30 Read block 152 len 20480 16/30 Write block 179 len 20480 data: 11 13 13 12 13 17/30 Read block 523 len 32768 18/30 Write block 481 len 4096 data: 15 19/30 Read block 396 len 8192 20/30 Write block 555 len 40960 data: 9 8 9 10 11 10 11 13 13 13 21/30 Read block 51 len 16384 22/30 Write block 421 len 8192 data: 17 15 23/30 Read block 50 len 32768 24/30 Flush 25/30 Write block 293 len 28672 data: 14 14 10 8 11 13 14 26/30 Read block 449 len 20480 27/30 Read block 414 len 28672 28/30 Write block 370 len 32768 data: 8 9 8 10 9 8 8 9 29/30 Write block 519 len 32768 data: 13 11 12 11 11 10 7 7 30/30 Read block 24 len 36864 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 5 6 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:33.585451888Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1827} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 265 265 265 F F F 1 020-039 159 159 159 265 265 265 F F F 2 040-059 157 157 157 261 261 261 F F F 3 060-079 159 159 159 265 265 265 F F F 4 080-099 156 156 156 260 260 260 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 155 155 155 259 259 259 F F F 7 140-159 160 160 160 266 266 266 F F F 8 160-179 160 160 160 271 271 271 F F F 9 180-199 160 160 160 271 271 271 F F F 10 200-219 159 159 159 265 265 265 F F F 11 220-239 158 158 158 263 263 263 F F F 12 240-259 160 160 160 271 271 271 F F F 13 260-279 160 160 160 267 267 267 F F F 14 280-299 157 157 157 261 261 261 T T T 15 300-319 158 158 158 262 262 262 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 154 154 154 257 257 257 F F F 18 360-379 160 160 160 269 269 269 T T T 19 380-399 158 158 158 264 264 264 F F F 20 400-419 158 158 158 264 264 264 F F F 21 420-439 160 160 160 271 271 271 F F F 22 440-459 158 158 158 263 263 263 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 160 160 160 271 271 271 F F F 25 500-519 159 159 159 265 265 265 T T T 26 520-539 158 158 158 263 263 263 T T T 27 540-559 160 160 160 271 271 271 F F F 28 560-579 160 160 160 271 271 271 F F F 29 580-599 158 158 158 264 264 264 F F F Max gen: 160, Max flush: 271 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 265 265 265 F F F 1 020-039 159 159 159 265 265 265 F F F 2 040-059 157 157 157 261 261 261 F F F 3 060-079 159 159 159 265 265 265 F F F 4 080-099 156 156 156 260 260 260 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 155 155 155 259 259 259 F F F 7 140-159 160 160 160 266 266 266 F F F 8 160-179 160 160 160 271 271 271 F F F 9 180-199 160 160 160 271 271 271 F F F 10 200-219 159 159 159 265 265 265 F F F 11 220-239 158 158 158 263 263 263 F F F 12 240-259 160 160 160 271 271 271 F F F 13 260-279 160 160 160 267 267 267 F F F 14 280-299 161 161 161 272 272 272 F F F 15 300-319 158 158 158 262 262 262 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 154 154 154 257 257 257 F F F 18 360-379 161 161 161 272 272 272 F F F 19 380-399 158 158 158 264 264 264 F F F 20 400-419 158 158 158 264 264 264 F F F 21 420-439 160 160 160 271 271 271 F F F 22 440-459 158 158 158 263 263 263 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 160 160 160 271 271 271 F F F 25 500-519 161 161 161 272 272 272 F F F 26 520-539 161 161 161 272 272 272 F F F 27 540-559 160 160 160 271 271 271 F F F 28 560-579 160 160 160 271 271 271 F F F 29 580-599 158 158 158 264 264 264 F F F Max gen: 161, Max flush: 272 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.860631731Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.906045972Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.906904032Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.906941447Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.906956666Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.906971134Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835} {"msg":"Crucible f776e1b2-4370-45b0-9c8e-4349f6a04be1 has session id: 07de02dc-ae3e-42f0-b81f-abb64b7ba85d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.907534578Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Upstairs opts: Upstairs UUID: f776e1b2-4370-45b0-9c8e-4349f6a04be1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.907578093Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Crucible stats registered with UUID: f776e1b2-4370-45b0-9c8e-4349f6a04be1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.907595341Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938139037Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938214608Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835} {"msg":"f776e1b2-4370-45b0-9c8e-4349f6a04be1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938240835Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938264762Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"0","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938308057Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"0","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938336134Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"1","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.93837358Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"1","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938409135Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"2","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938491146Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"2","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938545349Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938581705Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938617421Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"ds_connection connected from Ok(127.0.0.1:52951)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938648817Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40662)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.938682433Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60913)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.93871497Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.939585268Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"2","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.939626313Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"0","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.93965867Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"1","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.940317233Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"1","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.940348179Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"0","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.940369907Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"2","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.940867758Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.940981135Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"[0]R flush_numbers[0..12]: [265, 265, 261, 265, 260, 260, 259, 266, 271, 271, 265, 263]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941057716Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"[0]R generation[0..12]: [159, 159, 157, 159, 156, 156, 155, 160, 160, 160, 159, 158]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941078274Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941093052Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"[1]R flush_numbers[0..12]: [265, 265, 261, 265, 260, 260, 259, 266, 271, 271, 265, 263]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941107161Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"[1]R generation[0..12]: [159, 159, 157, 159, 156, 156, 155, 160, 160, 160, 159, 158]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941121169Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941135097Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"[2]R flush_numbers[0..12]: [265, 265, 261, 265, 260, 260, 259, 266, 271, 271, 265, 263]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941148956Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"[2]R generation[0..12]: [159, 159, 157, 159, 156, 156, 155, 160, 160, 160, 159, 158]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941162714Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941176622Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941189531Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941202929Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941220427Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941233826Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"":"downstairs","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941247064Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941259203Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"0","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941272531Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"1"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941294589Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"client":"2","":"downstairs","session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941307807Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941320076Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"f776e1b2-4370-45b0-9c8e-4349f6a04be1 is now active with session: 07de02dc-ae3e-42f0-b81f-abb64b7ba85d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941332884Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941344773Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835,"session_id":"07de02dc-ae3e-42f0-b81f-abb64b7ba85d"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:37.941356901Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 567 len 4096 data: 12 02/30 Read block 243 len 24576 03/30 Write block 494 len 16384 data: 13 12 14 15 04/30 Read block 258 len 24576 05/30 Flush 06/30 Write block 255 len 40960 data: 15 13 14 16 16 16 16 18 17 14 07/30 Write block 294 len 20480 data: 15 11 9 12 14 08/30 Write block 443 len 24576 data: 11 9 8 9 8 9 09/30 Read block 6 len 16384 10/30 Write block 342 len 36864 data: 16 14 14 14 16 15 14 11 12 11/30 Write block 175 len 16384 data: 7 9 11 11 12/30 Read block 18 len 20480 13/30 Write block 92 len 20480 data: 17 19 15 15 16 14/30 Read block 540 len 36864 15/30 Read block 73 len 28672 16/30 Write block 252 len 24576 data: 10 13 14 16 14 15 17/30 Read block 279 len 32768 18/30 Flush 19/30 Write block 387 len 28672 data: 7 7 12 10 10 8 7 20/30 Read block 375 len 20480 21/30 Read block 194 len 36864 22/30 Read block 470 len 40960 23/30 Write block 34 len 24576 data: 5 4 7 8 11 12 24/30 Read block 591 len 20480 25/30 Read block 398 len 32768 26/30 Write block 426 len 20480 data: 14 12 12 12 12 27/30 Write block 4 len 12288 data: 7 6 9 28/30 Read block 179 len 36864 29/30 Write block 94 len 8192 data: 16 16 30/30 Write block 431 len 32768 data: 9 12 11 10 10 10 13 13 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:37.962033309Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1835} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 265 265 265 T T T 1 020-039 159 159 159 265 265 265 T T T 2 040-059 157 157 157 261 261 261 F F F 3 060-079 159 159 159 265 265 265 F F F 4 080-099 162 162 162 274 274 274 T T T 5 100-119 156 156 156 260 260 260 F F F 6 120-139 155 155 155 259 259 259 F F F 7 140-159 160 160 160 266 266 266 F F F 8 160-179 162 162 162 274 274 274 F F F 9 180-199 160 160 160 271 271 271 F F F 10 200-219 159 159 159 265 265 265 F F F 11 220-239 158 158 158 263 263 263 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 162 162 162 274 274 274 F F F 14 280-299 162 162 162 274 274 274 F F F 15 300-319 158 158 158 262 262 262 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 162 162 162 274 274 274 F F F 18 360-379 161 161 161 272 272 272 F F F 19 380-399 158 158 158 264 264 264 T T T 20 400-419 158 158 158 264 264 264 F F F 21 420-439 160 160 160 271 271 271 T T T 22 440-459 162 162 162 274 274 274 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 162 162 162 273 273 273 F F F 25 500-519 161 161 161 272 272 272 F F F 26 520-539 161 161 161 272 272 272 F F F 27 540-559 160 160 160 271 271 271 F F F 28 560-579 162 162 162 273 273 273 F F F 29 580-599 158 158 158 264 264 264 F F F Max gen: 162, Max flush: 274 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 275 275 275 F F F 1 020-039 163 163 163 275 275 275 F F F 2 040-059 157 157 157 261 261 261 F F F 3 060-079 159 159 159 265 265 265 F F F 4 080-099 163 163 163 275 275 275 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 155 155 155 259 259 259 F F F 7 140-159 160 160 160 266 266 266 F F F 8 160-179 162 162 162 274 274 274 F F F 9 180-199 160 160 160 271 271 271 F F F 10 200-219 159 159 159 265 265 265 F F F 11 220-239 158 158 158 263 263 263 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 162 162 162 274 274 274 F F F 14 280-299 162 162 162 274 274 274 F F F 15 300-319 158 158 158 262 262 262 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 162 162 162 274 274 274 F F F 18 360-379 161 161 161 272 272 272 F F F 19 380-399 163 163 163 275 275 275 F F F 20 400-419 158 158 158 264 264 264 F F F 21 420-439 163 163 163 275 275 275 F F F 22 440-459 162 162 162 274 274 274 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 162 162 162 273 273 273 F F F 25 500-519 161 161 161 272 272 272 F F F 26 520-539 161 161 161 272 272 272 F F F 27 540-559 160 160 160 271 271 271 F F F 28 560-579 162 162 162 273 273 273 F F F 29 580-599 158 158 158 264 264 264 F F F Max gen: 163, Max flush: 275 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.24889229Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:42.291944647Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.292365408Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.324095205Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.324716273Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.324761168Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.324778546Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.324796073Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843} {"msg":"Crucible aa7c4653-f583-40ae-b9f9-39c6ae14bdbd has session id: 328325de-8036-44b3-a8a4-0b5fadf55fc4","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.325308574Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Upstairs opts: Upstairs UUID: aa7c4653-f583-40ae-b9f9-39c6ae14bdbd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.325351599Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Crucible stats registered with UUID: aa7c4653-f583-40ae-b9f9-39c6ae14bdbd","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.325374566Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.355958107Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.355990393Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843} {"msg":"aa7c4653-f583-40ae-b9f9-39c6ae14bdbd active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356061875Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356085302Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"0","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356113749Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"0","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356131477Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"1","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356149145Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"1","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356165843Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"2","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356183121Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"2","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356199489Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356216557Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356238695Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:54734)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356343502Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:36328)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356395646Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:44478)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.356433202Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.35713486Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"1","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.357176675Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"2","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.357300821Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"0","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.357841158Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"2","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.357884932Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"1","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.357982261Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"0","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358377415Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358464905Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"[0]R flush_numbers[0..12]: [275, 275, 261, 265, 275, 260, 259, 266, 274, 271, 265, 263]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.35859119Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"[0]R generation[0..12]: [163, 163, 157, 159, 163, 156, 155, 160, 162, 160, 159, 158]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358612158Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358627056Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"[1]R flush_numbers[0..12]: [275, 275, 261, 265, 275, 260, 259, 266, 274, 271, 265, 263]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358641084Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"[1]R generation[0..12]: [163, 163, 157, 159, 163, 156, 155, 160, 162, 160, 159, 158]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358658852Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358673591Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"[2]R flush_numbers[0..12]: [275, 275, 261, 265, 275, 260, 259, 266, 274, 271, 265, 263]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358687529Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"[2]R generation[0..12]: [163, 163, 157, 159, 163, 156, 155, 160, 162, 160, 159, 158]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358701377Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358716355Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358729644Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:57:42.358742942Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.35875975Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358772729Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"":"downstairs","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358785947Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358797906Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"0","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358811424Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"1","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358824743Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"client":"2","":"downstairs","session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358838051Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.35885048Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"aa7c4653-f583-40ae-b9f9-39c6ae14bdbd is now active with session: 328325de-8036-44b3-a8a4-0b5fadf55fc4","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358865158Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:42.358877407Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843,"session_id":"328325de-8036-44b3-a8a4-0b5fadf55fc4"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-13T15:57:42.358889855Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 178 len 8192 data: 12 12 03/30 Read block 261 len 32768 04/30 Flush 05/30 Write block 293 len 32768 data: 15 16 12 10 13 15 15 14 06/30 Write block 382 len 4096 data: 12 07/30 Read block 519 len 4096 08/30 Write block 506 len 36864 data: 12 11 13 12 10 10 10 8 10 09/30 Write block 580 len 16384 data: 10 10 10 12 10/30 Read block 109 len 36864 11/30 Read block 91 len 8192 12/30 Write block 158 len 32768 data: 14 15 17 18 22 19 18 16 13/30 Read block 28 len 32768 14/30 Read block 274 len 8192 15/30 Write block 174 len 32768 data: 8 8 10 12 13 13 14 14 16/30 Read block 9 len 16384 17/30 Write block 593 len 16384 data: 15 17 14 13 18/30 Write block 442 len 40960 data: 7 12 10 9 10 9 10 13 12 13 19/30 Read block 1 len 20480 20/30 Read block 44 len 32768 21/30 Write block 489 len 28672 data: 13 10 9 10 11 14 13 22/30 Write block 7 len 4096 data: 13 23/30 Write block 124 len 8192 data: 15 13 24/30 Write block 509 len 24576 data: 13 11 11 11 9 11 25/30 Read block 318 len 32768 26/30 Read block 423 len 4096 27/30 Read block 465 len 8192 28/30 Read block 298 len 32768 29/30 Write block 514 len 16384 data: 12 11 14 12 30/30 Read block 208 len 12288 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 8 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 4 Sent Sent Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 24 24 26 74 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:42.377750034Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1843} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 275 275 275 T T T 1 020-039 163 163 163 275 275 275 F F F 2 040-059 157 157 157 261 261 261 F F F 3 060-079 159 159 159 265 265 265 F F F 4 080-099 163 163 163 275 275 275 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 155 155 155 259 259 259 T T T 7 140-159 160 160 160 266 266 266 T T T 8 160-179 164 164 164 277 277 277 T T T 9 180-199 160 160 160 271 271 271 T T T 10 200-219 159 159 159 265 265 265 F F F 11 220-239 158 158 158 263 263 263 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 162 162 162 274 274 274 F F F 14 280-299 162 162 162 274 274 274 T T T 15 300-319 158 158 158 262 262 262 T T T 16 320-339 160 160 160 268 268 268 F F F 17 340-359 162 162 162 274 274 274 F F F 18 360-379 161 161 161 272 272 272 F F F 19 380-399 163 163 163 275 275 275 T T T 20 400-419 158 158 158 264 264 264 F F F 21 420-439 163 163 163 275 275 275 F F F 22 440-459 162 162 162 274 274 274 T T T 23 460-479 160 160 160 269 269 269 F F F 24 480-499 162 162 162 273 273 273 T T T 25 500-519 161 161 161 272 272 272 T T T 26 520-539 161 161 161 272 272 272 F F F 27 540-559 160 160 160 271 271 271 F F F 28 560-579 162 162 162 273 273 273 F F F 29 580-599 158 158 158 264 264 264 T T T Max gen: 164, Max flush: 277 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 278 278 278 F F F 1 020-039 163 163 163 275 275 275 F F F 2 040-059 157 157 157 261 261 261 F F F 3 060-079 159 159 159 265 265 265 F F F 4 080-099 163 163 163 275 275 275 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 165 165 165 278 278 278 F F F 7 140-159 165 165 165 278 278 278 F F F 8 160-179 165 165 165 278 278 278 F F F 9 180-199 165 165 165 278 278 278 F F F 10 200-219 159 159 159 265 265 265 F F F 11 220-239 158 158 158 263 263 263 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 162 162 162 274 274 274 F F F 14 280-299 165 165 165 278 278 278 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 162 162 162 274 274 274 F F F 18 360-379 161 161 161 272 272 272 F F F 19 380-399 165 165 165 278 278 278 F F F 20 400-419 158 158 158 264 264 264 F F F 21 420-439 163 163 163 275 275 275 F F F 22 440-459 165 165 165 278 278 278 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 165 165 165 278 278 278 F F F 25 500-519 165 165 165 278 278 278 F F F 26 520-539 161 161 161 272 272 272 F F F 27 540-559 160 160 160 271 271 271 F F F 28 560-579 162 162 162 273 273 273 F F F 29 580-599 165 165 165 278 278 278 F F F Max gen: 165, Max flush: 278 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.726960471Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.773137363Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.774035698Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.774082853Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.774097441Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.77411157Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851} {"msg":"Crucible de25964b-60e1-47d0-8b90-582ee073f397 has session id: ade34b31-f115-4f85-8c1b-9771adb57df1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.774642537Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Upstairs opts: Upstairs UUID: de25964b-60e1-47d0-8b90-582ee073f397, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.774686842Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Crucible stats registered with UUID: de25964b-60e1-47d0-8b90-582ee073f397","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.77470534Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805500826Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805559909Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851} {"msg":"de25964b-60e1-47d0-8b90-582ee073f397 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805578087Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805594905Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"0","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805626432Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"0","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805652489Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"1","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805670916Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"1","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805688534Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"2","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805707202Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"2","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.80572398Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805742438Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805759226Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45443)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805822558Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"ds_connection connected from Ok(127.0.0.1:56598)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.805959673Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46595)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.806000568Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.806601988Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"0","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.806634234Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"1","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.807215546Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"2","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.807253662Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"0","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.807282438Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"1","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.80778261Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"2","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.807917924Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.807988316Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"[0]R flush_numbers[0..12]: [278, 275, 261, 265, 275, 260, 278, 278, 278, 278, 265, 263]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808373001Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"[0]R generation[0..12]: [165, 163, 157, 159, 163, 156, 165, 165, 165, 165, 159, 158]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808398258Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808413196Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"[1]R flush_numbers[0..12]: [278, 275, 261, 265, 275, 260, 278, 278, 278, 278, 265, 263]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808427215Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"[1]R generation[0..12]: [165, 163, 157, 159, 163, 156, 165, 165, 165, 165, 159, 158]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808441153Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808455141Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"[2]R flush_numbers[0..12]: [278, 275, 261, 265, 275, 260, 278, 278, 278, 278, 265, 263]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.80846897Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"[2]R generation[0..12]: [165, 163, 157, 159, 163, 156, 165, 165, 165, 165, 159, 158]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808483218Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808497417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808510505Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808532872Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808547081Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808560069Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"":"downstairs","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808580397Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808592705Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"0","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808605914Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"1","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808618922Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"client":"2","":"downstairs","session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808632281Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808644469Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"de25964b-60e1-47d0-8b90-582ee073f397 is now active with session: ade34b31-f115-4f85-8c1b-9771adb57df1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808657298Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808669306Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851,"session_id":"ade34b31-f115-4f85-8c1b-9771adb57df1"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:46.808681595Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 74 len 4096 data: 12 03/30 Write block 175 len 20480 data: 9 11 13 14 14 04/30 Read block 219 len 28672 05/30 Write block 394 len 8192 data: 8 7 06/30 Flush 07/30 Write block 587 len 20480 data: 11 10 11 12 14 08/30 Flush 09/30 Write block 76 len 4096 data: 5 10/30 Read block 227 len 24576 11/30 Flush 12/30 Write block 359 len 24576 data: 15 13 13 14 13 12 13/30 Flush 14/30 Write block 167 len 20480 data: 17 17 14 15 11 15/30 Flush 16/30 Flush 17/30 Write block 93 len 12288 data: 20 17 17 18/30 Read block 255 len 28672 19/30 Read block 315 len 8192 20/30 Write block 41 len 28672 data: 12 13 16 15 13 15 12 21/30 Read block 178 len 24576 22/30 Read block 401 len 40960 23/30 Read block 433 len 36864 24/30 Read block 212 len 16384 25/30 Read block 207 len 36864 26/30 Write block 563 len 32768 data: 14 14 14 15 13 12 14 13 27/30 Read block 12 len 12288 28/30 Read block 552 len 40960 29/30 Write block 218 len 32768 data: 16 13 13 13 13 12 13 9 30/30 Read block 116 len 28672 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 13 14 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:46.829359212Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1851} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 278 278 278 F F F 1 020-039 163 163 163 275 275 275 F F F 2 040-059 157 157 157 261 261 261 T T T 3 060-079 166 166 166 282 282 282 F F F 4 080-099 163 163 163 275 275 275 T T T 5 100-119 156 156 156 260 260 260 F F F 6 120-139 165 165 165 278 278 278 F F F 7 140-159 165 165 165 278 278 278 F F F 8 160-179 166 166 166 284 284 284 F F F 9 180-199 165 165 165 278 278 278 F F F 10 200-219 159 159 159 265 265 265 T T T 11 220-239 158 158 158 263 263 263 T T T 12 240-259 162 162 162 274 274 274 F F F 13 260-279 162 162 162 274 274 274 F F F 14 280-299 165 165 165 278 278 278 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 166 166 166 283 283 283 F F F 18 360-379 166 166 166 283 283 283 F F F 19 380-399 166 166 166 280 280 280 F F F 20 400-419 158 158 158 264 264 264 F F F 21 420-439 163 163 163 275 275 275 F F F 22 440-459 165 165 165 278 278 278 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 165 165 165 278 278 278 F F F 25 500-519 165 165 165 278 278 278 F F F 26 520-539 161 161 161 272 272 272 F F F 27 540-559 160 160 160 271 271 271 F F F 28 560-579 162 162 162 273 273 273 T T T 29 580-599 166 166 166 281 281 281 F F F Max gen: 166, Max flush: 284 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 278 278 278 F F F 1 020-039 163 163 163 275 275 275 F F F 2 040-059 167 167 167 285 285 285 F F F 3 060-079 166 166 166 282 282 282 F F F 4 080-099 167 167 167 285 285 285 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 165 165 165 278 278 278 F F F 7 140-159 165 165 165 278 278 278 F F F 8 160-179 166 166 166 284 284 284 F F F 9 180-199 165 165 165 278 278 278 F F F 10 200-219 167 167 167 285 285 285 F F F 11 220-239 167 167 167 285 285 285 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 162 162 162 274 274 274 F F F 14 280-299 165 165 165 278 278 278 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 166 166 166 283 283 283 F F F 18 360-379 166 166 166 283 283 283 F F F 19 380-399 166 166 166 280 280 280 F F F 20 400-419 158 158 158 264 264 264 F F F 21 420-439 163 163 163 275 275 275 F F F 22 440-459 165 165 165 278 278 278 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 165 165 165 278 278 278 F F F 25 500-519 165 165 165 278 278 278 F F F 26 520-539 161 161 161 272 272 272 F F F 27 540-559 160 160 160 271 271 271 F F F 28 560-579 167 167 167 285 285 285 F F F 29 580-599 166 166 166 281 281 281 F F F Max gen: 167, Max flush: 285 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.113740733Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:51.157103413Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.157705223Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.189204817Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.190114651Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.190155636Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.190171374Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.190186273Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859} {"msg":"Crucible 4f684844-8693-490a-9957-b735c4b9f56c has session id: 11ec8adc-5a43-4071-bbbb-92e26748a40d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.190735109Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Upstairs opts: Upstairs UUID: 4f684844-8693-490a-9957-b735c4b9f56c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.190787082Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Crucible stats registered with UUID: 4f684844-8693-490a-9957-b735c4b9f56c","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.190819209Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.221267796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.221326589Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859} {"msg":"4f684844-8693-490a-9957-b735c4b9f56c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.221342717Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.221372894Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"0","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.22140206Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"0","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.221420238Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"1","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.221438366Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"1","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.221455404Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"2","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.221474791Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"2","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.22149151Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.221508998Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.221526065Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62880)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.221809423Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41601)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.22200422Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45259)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.222040856Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.222683901Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"1","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.222740344Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"2","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.222801827Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"0","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.223600783Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"1","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.223640899Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"2","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.223726469Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"0","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224196204Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224221051Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"[0]R flush_numbers[0..12]: [278, 275, 285, 282, 285, 260, 278, 278, 284, 278, 285, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224308371Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"[0]R generation[0..12]: [165, 163, 167, 166, 167, 156, 165, 165, 166, 165, 167, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224328419Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224343057Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"[1]R flush_numbers[0..12]: [278, 275, 285, 282, 285, 260, 278, 278, 284, 278, 285, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224358535Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"[1]R generation[0..12]: [165, 163, 167, 166, 167, 156, 165, 165, 166, 165, 167, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224376993Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224391771Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"[2]R flush_numbers[0..12]: [278, 275, 285, 282, 285, 260, 278, 278, 284, 278, 285, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224405769Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"[2]R generation[0..12]: [165, 163, 167, 166, 167, 156, 165, 165, 166, 165, 167, 167]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224419538Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224433546Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224446575Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224460043Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224473462Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.22448664Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"":"downstairs","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224499978Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":Wait for a query_work_queue command to finish before sending IO 1859,"session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.22457597Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"0","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224593128Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"1","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224606766Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"client":"2","":"downstairs","session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224620635Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224633993Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"4f684844-8693-490a-9957-b735c4b9f56c is now active with session: 11ec8adc-5a43-4071-bbbb-92e26748a40d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224650061Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.22466258Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859,"session_id":"11ec8adc-5a43-4071-bbbb-92e26748a40d"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:51.224675098Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 522 len 32768 data: 12 12 11 8 8 8 9 10 03/30 Read block 207 len 4096 04/30 Read block 568 len 36864 05/30 Flush 06/30 Write block 485 len 40960 data: 10 12 13 14 14 11 10 11 12 15 07/30 Read block 556 len 16384 08/30 Flush 09/30 Write block 123 len 32768 data: 16 16 14 16 18 16 12 8 10/30 Read block 128 len 12288 11/30 Read block 85 len 20480 12/30 Read block 310 len 24576 13/30 Read block 52 len 8192 14/30 Write block 356 len 12288 data: 9 10 12 15/30 Read block 264 len 28672 16/30 Write block 292 len 24576 data: 16 16 17 13 11 14 17/30 Write block 485 len 4096 data: 11 18/30 Flush 19/30 Write block 207 len 12288 data: 16 13 15 20/30 Write block 396 len 28672 data: 9 16 14 15 14 15 18 21/30 Write block 225 len 32768 data: 10 11 12 16 15 18 17 17 22/30 Write block 499 len 20480 data: 15 18 17 17 17 23/30 Read block 561 len 36864 24/30 Read block 245 len 24576 25/30 Read block 246 len 24576 26/30 Write block 403 len 24576 data: 19 17 16 17 13 15 27/30 Write block 148 len 24576 data: 16 15 13 15 14 15 28/30 Write block 196 len 36864 data: 12 12 12 13 16 14 14 14 16 29/30 Write block 568 len 12288 data: 13 15 14 30/30 Read block 161 len 16384 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 3 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 12 10 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:51.245436286Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1859} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 278 278 278 F F F 1 020-039 163 163 163 275 275 275 F F F 2 040-059 167 167 167 285 285 285 F F F 3 060-079 166 166 166 282 282 282 F F F 4 080-099 167 167 167 285 285 285 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 168 168 168 289 289 289 F F F 7 140-159 165 165 165 278 278 278 T T T 8 160-179 166 166 166 284 284 284 F F F 9 180-199 165 165 165 278 278 278 T T T 10 200-219 167 167 167 285 285 285 T T T 11 220-239 167 167 167 285 285 285 T T T 12 240-259 162 162 162 274 274 274 F F F 13 260-279 162 162 162 274 274 274 F F F 14 280-299 168 168 168 289 289 289 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 168 168 168 289 289 289 F F F 18 360-379 166 166 166 283 283 283 F F F 19 380-399 166 166 166 280 280 280 T T T 20 400-419 158 158 158 264 264 264 T T T 21 420-439 163 163 163 275 275 275 F F F 22 440-459 165 165 165 278 278 278 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 168 168 168 289 289 289 T T T 25 500-519 165 165 165 278 278 278 T T T 26 520-539 168 168 168 287 287 287 F F F 27 540-559 160 160 160 271 271 271 F F F 28 560-579 167 167 167 285 285 285 T T T 29 580-599 166 166 166 281 281 281 F F F Max gen: 168, Max flush: 289 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 278 278 278 F F F 1 020-039 163 163 163 275 275 275 F F F 2 040-059 167 167 167 285 285 285 F F F 3 060-079 166 166 166 282 282 282 F F F 4 080-099 167 167 167 285 285 285 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 168 168 168 289 289 289 F F F 7 140-159 169 169 169 290 290 290 F F F 8 160-179 166 166 166 284 284 284 F F F 9 180-199 169 169 169 290 290 290 F F F 10 200-219 169 169 169 290 290 290 F F F 11 220-239 169 169 169 290 290 290 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 162 162 162 274 274 274 F F F 14 280-299 168 168 168 289 289 289 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 168 168 168 289 289 289 F F F 18 360-379 166 166 166 283 283 283 F F F 19 380-399 169 169 169 290 290 290 F F F 20 400-419 169 169 169 290 290 290 F F F 21 420-439 163 163 163 275 275 275 F F F 22 440-459 165 165 165 278 278 278 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 169 169 169 290 290 290 F F F 25 500-519 169 169 169 290 290 290 F F F 26 520-539 168 168 168 287 287 287 F F F 27 540-559 160 160 160 271 271 271 F F F 28 560-579 169 169 169 290 290 290 F F F 29 580-599 166 166 166 281 281 281 F F F Max gen: 169, Max flush: 290 Begin loop 85 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.592345161Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.637207526Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.638042898Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.638089963Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.63811301Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.638131458Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867} {"msg":"Crucible 99196f16-3452-4b15-9395-3c64480f2454 has session id: c226fba1-75b8-4156-a738-8654bbe0c06c","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.638557289Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Upstairs opts: Upstairs UUID: 99196f16-3452-4b15-9395-3c64480f2454, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.638611262Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Crucible stats registered with UUID: 99196f16-3452-4b15-9395-3c64480f2454","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.638641139Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.668803949Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.668858493Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867} {"msg":"99196f16-3452-4b15-9395-3c64480f2454 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.668892439Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.668922355Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"0","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.66896979Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"0","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.669008115Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"1","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.669043681Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"1","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.669079077Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"2","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.669122032Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"2","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.669153188Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.669180405Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.669207792Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34409)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.669234969Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35505)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.669263875Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"ds_connection connected from Ok(127.0.0.1:50315)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.669291852Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.670091469Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"2","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.670239772Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"0","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.670360058Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"1","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.670760981Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"2","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.670803036Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"0","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.670930391Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"1","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671400346Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671435782Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"[0]R flush_numbers[0..12]: [278, 275, 285, 282, 285, 260, 289, 290, 284, 290, 290, 290]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671546669Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"[0]R generation[0..12]: [165, 163, 167, 166, 167, 156, 168, 169, 166, 169, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671566747Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671581495Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"[1]R flush_numbers[0..12]: [278, 275, 285, 282, 285, 260, 289, 290, 284, 290, 290, 290]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671595443Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"[1]R generation[0..12]: [165, 163, 167, 166, 167, 156, 168, 169, 166, 169, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671609302Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.67162462Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"[2]R flush_numbers[0..12]: [278, 275, 285, 282, 285, 260, 289, 290, 284, 290, 290, 290]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671638609Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"[2]R generation[0..12]: [165, 163, 167, 166, 167, 156, 168, 169, 166, 169, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671652547Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671666605Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671679504Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671692612Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.67170565Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671718639Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"":"downstairs","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671741576Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671753955Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"0","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671767223Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"1","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671780402Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"client":"2","":"downstairs","session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.67179359Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671806059Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"99196f16-3452-4b15-9395-3c64480f2454 is now active with session: c226fba1-75b8-4156-a738-8654bbe0c06c","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671818907Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671830856Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867,"session_id":"c226fba1-75b8-4156-a738-8654bbe0c06c"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:55.671843094Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 233 len 12288 data: 11 11 10 02/30 Read block 519 len 4096 03/30 Read block 218 len 8192 04/30 Read block 117 len 36864 05/30 Read block 158 len 4096 06/30 Read block 254 len 40960 07/30 Write block 291 len 20480 data: 15 17 17 18 14 08/30 Write block 224 len 8192 data: 14 11 09/30 Write block 272 len 20480 data: 7 5 7 8 10 10/30 Read block 442 len 12288 11/30 Read block 449 len 12288 12/30 Write block 87 len 20480 data: 14 15 20 17 19 13/30 Flush 14/30 Write block 292 len 4096 data: 18 15/30 Write block 38 len 16384 data: 12 13 10 13 16/30 Read block 65 len 20480 17/30 Read block 72 len 20480 18/30 Read block 525 len 32768 19/30 Write block 33 len 20480 data: 6 6 5 8 9 20/30 Read block 124 len 36864 21/30 Read block 64 len 24576 22/30 Read block 448 len 24576 23/30 Write block 348 len 40960 data: 15 12 13 11 10 7 12 10 10 11 24/30 Write block 387 len 8192 data: 8 8 25/30 Flush 26/30 Write block 553 len 32768 data: 11 7 10 9 10 11 12 11 27/30 Read block 427 len 32768 28/30 Write block 579 len 8192 data: 8 11 29/30 Read block 147 len 24576 30/30 Write block 497 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:57:55.691365047Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1867} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 278 278 278 F F F 1 020-039 170 170 170 292 292 292 F F F 2 040-059 170 170 170 292 292 292 F F F 3 060-079 166 166 166 282 282 282 F F F 4 080-099 170 170 170 291 291 291 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 168 168 168 289 289 289 F F F 7 140-159 169 169 169 290 290 290 F F F 8 160-179 166 166 166 284 284 284 F F F 9 180-199 169 169 169 290 290 290 F F F 10 200-219 169 169 169 290 290 290 F F F 11 220-239 170 170 170 291 291 291 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 170 170 170 291 291 291 F F F 14 280-299 170 170 170 292 292 292 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 170 170 170 292 292 292 F F F 18 360-379 166 166 166 283 283 283 F F F 19 380-399 170 170 170 292 292 292 F F F 20 400-419 169 169 169 290 290 290 F F F 21 420-439 163 163 163 275 275 275 F F F 22 440-459 165 165 165 278 278 278 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 169 169 169 290 290 290 T T T 25 500-519 169 169 169 290 290 290 F F F 26 520-539 168 168 168 287 287 287 F F F 27 540-559 160 160 160 271 271 271 T T T 28 560-579 169 169 169 290 290 290 T T T 29 580-599 166 166 166 281 281 281 T T T Max gen: 170, Max flush: 292 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 278 278 278 F F F 1 020-039 170 170 170 292 292 292 F F F 2 040-059 170 170 170 292 292 292 F F F 3 060-079 166 166 166 282 282 282 F F F 4 080-099 170 170 170 291 291 291 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 168 168 168 289 289 289 F F F 7 140-159 169 169 169 290 290 290 F F F 8 160-179 166 166 166 284 284 284 F F F 9 180-199 169 169 169 290 290 290 F F F 10 200-219 169 169 169 290 290 290 F F F 11 220-239 170 170 170 291 291 291 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 170 170 170 291 291 291 F F F 14 280-299 170 170 170 292 292 292 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 170 170 170 292 292 292 F F F 18 360-379 166 166 166 283 283 283 F F F 19 380-399 170 170 170 292 292 292 F F F 20 400-419 169 169 169 290 290 290 F F F 21 420-439 163 163 163 275 275 275 F F F 22 440-459 165 165 165 278 278 278 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 171 171 171 293 293 293 F F F 25 500-519 169 169 169 290 290 290 F F F 26 520-539 168 168 168 287 287 287 F F F 27 540-559 171 171 171 293 293 293 F F F 28 560-579 171 171 171 293 293 293 F F F 29 580-599 171 171 171 293 293 293 F F F Max gen: 171, Max flush: 293 Begin loop 86 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:57:59.965217535Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.010930051Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.011586275Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.01162512Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.011639338Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.011654397Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875} {"msg":"Crucible b394c680-4721-4528-a449-7f97a12f674d has session id: 4bb8c243-845b-477c-a473-87f9393825ca","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.012067998Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Upstairs opts: Upstairs UUID: b394c680-4721-4528-a449-7f97a12f674d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.012114083Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Crucible stats registered with UUID: b394c680-4721-4528-a449-7f97a12f674d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.01213864Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042030562Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042065998Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875} {"msg":"b394c680-4721-4528-a449-7f97a12f674d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042099784Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042117672Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"0","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042147189Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"0","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042176985Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"1","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042212961Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"1","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042247097Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"2","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042284442Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"2","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042319809Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042356914Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.04239379Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:39438)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042430396Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:33854)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042463942Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:52689)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.042615364Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.043157451Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"1","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.043188637Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"0","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.043205155Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"2","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.04393708Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"1","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044097171Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"0","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044141696Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"2","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044420413Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044605792Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"[0]R flush_numbers[0..12]: [278, 292, 292, 282, 291, 260, 289, 290, 284, 290, 290, 291]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044730377Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"[0]R generation[0..12]: [165, 170, 170, 166, 170, 156, 168, 169, 166, 169, 169, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044761473Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044778791Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"[1]R flush_numbers[0..12]: [278, 292, 292, 282, 291, 260, 289, 290, 284, 290, 290, 291]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.04479324Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"[1]R generation[0..12]: [165, 170, 170, 166, 170, 156, 168, 169, 166, 169, 169, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044807618Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044821877Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"[2]R flush_numbers[0..12]: [278, 292, 292, 282, 291, 260, 289, 290, 284, 290, 290, 291]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044835585Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"[2]R generation[0..12]: [165, 170, 170, 166, 170, 156, 168, 169, 166, 169, 169, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044849423Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044863772Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.04487698Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044893708Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Next flush: 294"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044914726Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044939933Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"":"downstairs","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.044967889Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.045010064Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"0","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.045042841Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"1","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.045072567Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"client":"2","":"downstairs","session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.045101424Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.045127781Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"b394c680-4721-4528-a449-7f97a12f674d is now active with session: 4bb8c243-845b-477c-a473-87f9393825ca","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-13T15:58:00.045155907Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.045182255Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875,"session_id":"4bb8c243-845b-477c-a473-87f9393825ca"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:00.045207812Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 582 len 28672 data: 11 13 9 10 8 12 11 02/30 Read block 561 len 36864 03/30 Read block 431 len 16384 04/30 Read block 535 len 8192 05/30 Read block 377 len 24576 06/30 Read block 92 len 36864 07/30 Write block 445 len 4096 data: 10 08/30 Read block 333 len 12288 09/30 Read block 222 len 28672 10/30 Read block 520 len 8192 11/30 Read block 447 len 36864 12/30 Flush 13/30 Write block 512 len 16384 data: 12 10 13 12 14/30 Write block 430 len 32768 data: 13 10 13 12 11 11 11 14 15/30 Read block 364 len 16384 16/30 Flush 17/30 Write block 93 len 12288 data: 21 18 18 18/30 Read block 115 len 12288 19/30 Read block 30 len 32768 20/30 Read block 517 len 12288 21/30 Read block 58 len 24576 22/30 Write block 532 len 12288 data: 16 15 15 23/30 Write block 440 len 28672 data: 9 8 8 13 11 11 11 24/30 Write block 182 len 16384 data: 13 14 13 11 25/30 Flush 26/30 Write block 28 len 12288 data: 14 10 8 27/30 Write block 596 len 4096 data: 14 28/30 Write block 425 len 32768 data: 13 15 13 13 13 14 11 14 29/30 Write block 418 len 40960 data: 13 17 17 18 16 16 15 14 16 14 30/30 Write block 505 len 40960 data: 13 13 12 14 14 12 12 13 11 14 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:00.06433065Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1875} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 278 278 278 F F F 1 020-039 170 170 170 292 292 292 T T T 2 040-059 170 170 170 292 292 292 F F F 3 060-079 166 166 166 282 282 282 F F F 4 080-099 172 172 172 296 296 296 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 168 168 168 289 289 289 F F F 7 140-159 169 169 169 290 290 290 F F F 8 160-179 166 166 166 284 284 284 F F F 9 180-199 172 172 172 296 296 296 F F F 10 200-219 169 169 169 290 290 290 F F F 11 220-239 170 170 170 291 291 291 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 170 170 170 291 291 291 F F F 14 280-299 170 170 170 292 292 292 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 170 170 170 292 292 292 F F F 18 360-379 166 166 166 283 283 283 F F F 19 380-399 170 170 170 292 292 292 F F F 20 400-419 169 169 169 290 290 290 T T T 21 420-439 172 172 172 295 295 295 T T T 22 440-459 172 172 172 296 296 296 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 171 171 171 293 293 293 F F F 25 500-519 172 172 172 295 295 295 T T T 26 520-539 172 172 172 296 296 296 F F F 27 540-559 171 171 171 293 293 293 F F F 28 560-579 171 171 171 293 293 293 F F F 29 580-599 172 172 172 294 294 294 T T T Max gen: 172, Max flush: 296 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 278 278 278 F F F 1 020-039 173 173 173 297 297 297 F F F 2 040-059 170 170 170 292 292 292 F F F 3 060-079 166 166 166 282 282 282 F F F 4 080-099 172 172 172 296 296 296 F F F 5 100-119 156 156 156 260 260 260 F F F 6 120-139 168 168 168 289 289 289 F F F 7 140-159 169 169 169 290 290 290 F F F 8 160-179 166 166 166 284 284 284 F F F 9 180-199 172 172 172 296 296 296 F F F 10 200-219 169 169 169 290 290 290 F F F 11 220-239 170 170 170 291 291 291 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 170 170 170 291 291 291 F F F 14 280-299 170 170 170 292 292 292 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 160 160 160 268 268 268 F F F 17 340-359 170 170 170 292 292 292 F F F 18 360-379 166 166 166 283 283 283 F F F 19 380-399 170 170 170 292 292 292 F F F 20 400-419 173 173 173 297 297 297 F F F 21 420-439 173 173 173 297 297 297 F F F 22 440-459 172 172 172 296 296 296 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 171 171 171 293 293 293 F F F 25 500-519 173 173 173 297 297 297 F F F 26 520-539 172 172 172 296 296 296 F F F 27 540-559 171 171 171 293 293 293 F F F 28 560-579 171 171 171 293 293 293 F F F 29 580-599 173 173 173 297 297 297 F F F Max gen: 173, Max flush: 297 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.335138255Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.39003119Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.390935424Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.39097394Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.390989088Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.391004296Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883} {"msg":"Crucible 09300bcd-52af-4e6f-adb4-ad93dfb22026 has session id: f55f115f-fdaa-47d4-8c1c-145bece34cce","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.391495999Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Upstairs opts: Upstairs UUID: 09300bcd-52af-4e6f-adb4-ad93dfb22026, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.391542083Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Crucible stats registered with UUID: 09300bcd-52af-4e6f-adb4-ad93dfb22026","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.39157205Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.421590917Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.421632232Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883} {"msg":"09300bcd-52af-4e6f-adb4-ad93dfb22026 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.421695925Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.421731691Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"0","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.421764896Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"0","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.421796943Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"1","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.421831999Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"1","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.421864205Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"2","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.421901261Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"2","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.421934077Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.421969173Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.422003349Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"ds_connection connected from Ok(127.0.0.1:58254)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.422037545Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"ds_connection connected from Ok(127.0.0.1:50328)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.422071231Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"ds_connection connected from Ok(127.0.0.1:46689)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.422104947Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.422712276Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"2","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.42275994Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"0","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.423014881Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"1","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.423506293Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"0","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.423547029Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"2","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.423615891Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"1","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424172516Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424281083Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"[0]R flush_numbers[0..12]: [278, 297, 292, 282, 296, 260, 289, 290, 284, 296, 290, 291]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424347345Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"[0]R generation[0..12]: [165, 173, 170, 166, 172, 156, 168, 169, 166, 172, 169, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424368073Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424382981Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"[1]R flush_numbers[0..12]: [278, 297, 292, 282, 296, 260, 289, 290, 284, 296, 290, 291]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424397309Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"[1]R generation[0..12]: [165, 173, 170, 166, 172, 156, 168, 169, 166, 172, 169, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424411448Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424425686Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"[2]R flush_numbers[0..12]: [278, 297, 292, 282, 296, 260, 289, 290, 284, 296, 290, 291]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424445044Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"[2]R generation[0..12]: [165, 173, 170, 166, 172, 156, 168, 169, 166, 172, 169, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424459952Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424474151Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424487349Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424500787Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424513866Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424527014Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"":"downstairs","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:58:04.424540283Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424557731Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"0","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424572249Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"1","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424585687Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"client":"2","":"downstairs","session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424599236Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424611514Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"09300bcd-52af-4e6f-adb4-ad93dfb22026 is now active with session: f55f115f-fdaa-47d4-8c1c-145bece34cce","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424624393Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.424636592Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883,"session_id":"f55f115f-fdaa-47d4-8c1c-145bece34cce"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:04.42464879Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 33 len 16384 data: 7 7 6 9 02/30 Read block 123 len 12288 03/30 Read block 191 len 8192 04/30 Write block 114 len 16384 data: 11 10 17 19 05/30 Read block 427 len 40960 06/30 Read block 67 len 16384 07/30 Write block 561 len 16384 data: 12 14 15 15 08/30 Read block 312 len 4096 09/30 Write block 398 len 40960 data: 15 16 15 16 19 20 18 17 18 14 10/30 Write block 82 len 8192 data: 12 10 11/30 Flush 12/30 Write block 519 len 40960 data: 14 12 13 13 13 12 9 9 9 10 13/30 Read block 488 len 16384 14/30 Read block 310 len 32768 15/30 Write block 399 len 12288 data: 17 16 17 16/30 Write block 168 len 4096 data: 18 17/30 Write block 338 len 40960 data: 13 11 12 14 17 15 15 15 17 16 18/30 Read block 184 len 36864 19/30 Write block 138 len 12288 data: 11 10 15 20/30 Read block 349 len 28672 21/30 Read block 358 len 24576 22/30 Write block 383 len 16384 data: 11 12 10 10 23/30 Write block 42 len 28672 data: 14 17 16 14 16 13 11 24/30 Read block 304 len 36864 25/30 Write block 361 len 20480 data: 14 15 14 13 13 26/30 Write block 372 len 40960 data: 9 11 10 9 9 10 9 9 10 11 27/30 Write block 144 len 4096 data: 15 28/30 Read block 404 len 12288 29/30 Write block 214 len 40960 data: 15 16 15 14 17 14 14 14 14 13 30/30 Write block 215 len 20480 data: 17 16 15 18 15 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 10 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:04.444772692Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1883} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 278 278 278 F F F 1 020-039 174 174 174 298 298 298 F F F 2 040-059 170 170 170 292 292 292 T T T 3 060-079 166 166 166 282 282 282 F F F 4 080-099 174 174 174 298 298 298 F F F 5 100-119 174 174 174 298 298 298 F F F 6 120-139 168 168 168 289 289 289 T T T 7 140-159 169 169 169 290 290 290 T T T 8 160-179 166 166 166 284 284 284 T T T 9 180-199 172 172 172 296 296 296 F F F 10 200-219 169 169 169 290 290 290 T T T 11 220-239 170 170 170 291 291 291 T T T 12 240-259 162 162 162 274 274 274 F F F 13 260-279 170 170 170 291 291 291 F F F 14 280-299 170 170 170 292 292 292 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 160 160 160 268 268 268 T T T 17 340-359 170 170 170 292 292 292 T T T 18 360-379 166 166 166 283 283 283 T T T 19 380-399 174 174 174 298 298 298 T T T 20 400-419 174 174 174 298 298 298 T T T 21 420-439 173 173 173 297 297 297 F F F 22 440-459 172 172 172 296 296 296 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 171 171 171 293 293 293 F F F 25 500-519 173 173 173 297 297 297 T T T 26 520-539 172 172 172 296 296 296 T T T 27 540-559 171 171 171 293 293 293 F F F 28 560-579 174 174 174 298 298 298 F F F 29 580-599 173 173 173 297 297 297 F F F Max gen: 174, Max flush: 298 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 278 278 278 F F F 1 020-039 174 174 174 298 298 298 F F F 2 040-059 175 175 175 299 299 299 F F F 3 060-079 166 166 166 282 282 282 F F F 4 080-099 174 174 174 298 298 298 F F F 5 100-119 174 174 174 298 298 298 F F F 6 120-139 175 175 175 299 299 299 F F F 7 140-159 175 175 175 299 299 299 F F F 8 160-179 175 175 175 299 299 299 F F F 9 180-199 172 172 172 296 296 296 F F F 10 200-219 175 175 175 299 299 299 F F F 11 220-239 175 175 175 299 299 299 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 170 170 170 291 291 291 F F F 14 280-299 170 170 170 292 292 292 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 175 175 175 299 299 299 F F F 17 340-359 175 175 175 299 299 299 F F F 18 360-379 175 175 175 299 299 299 F F F 19 380-399 175 175 175 299 299 299 F F F 20 400-419 175 175 175 299 299 299 F F F 21 420-439 173 173 173 297 297 297 F F F 22 440-459 172 172 172 296 296 296 F F F 23 460-479 160 160 160 269 269 269 F F F 24 480-499 171 171 171 293 293 293 F F F 25 500-519 175 175 175 299 299 299 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 171 171 171 293 293 293 F F F 28 560-579 174 174 174 298 298 298 F F F 29 580-599 173 173 173 297 297 297 F F F Max gen: 175, Max flush: 299 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.77134247Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.81666547Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.817388996Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.817433931Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.817453608Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.817469306Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891} {"msg":"Crucible 283bf56d-d3ce-4534-8031-3a49815e4fef has session id: 854808b7-19ea-4851-9a92-2cb68d900fbe","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.817817586Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Upstairs opts: Upstairs UUID: 283bf56d-d3ce-4534-8031-3a49815e4fef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.817862621Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Crucible stats registered with UUID: 283bf56d-d3ce-4534-8031-3a49815e4fef","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.817886398Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.848660777Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.848698452Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891} {"msg":"283bf56d-d3ce-4534-8031-3a49815e4fef active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.848749506Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.848770264Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"0","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.848799171Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"0","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.848816809Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"1","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.848835596Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"1","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.848852534Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"2","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.848870512Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"2","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.84889151Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.848909018Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.848926256Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"ds_connection connected from Ok(127.0.0.1:56910)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.849076588Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"ds_connection connected from Ok(127.0.0.1:50008)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.849310241Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"ds_connection connected from Ok(127.0.0.1:56839)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.849339768Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.849952686Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"0","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.850015569Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"1","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.850100689Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"2","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.850695319Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"1","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.850725836Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"2","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.850783789Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"0","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851272432Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.85137277Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"[0]R flush_numbers[0..12]: [278, 298, 299, 282, 298, 298, 299, 299, 299, 296, 299, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851483067Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"[0]R generation[0..12]: [165, 174, 175, 166, 174, 174, 175, 175, 175, 172, 175, 175]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851506085Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851521373Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"[1]R flush_numbers[0..12]: [278, 298, 299, 282, 298, 298, 299, 299, 299, 296, 299, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851535641Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"[1]R generation[0..12]: [165, 174, 175, 166, 174, 174, 175, 175, 175, 172, 175, 175]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.85154959Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851564158Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"[2]R flush_numbers[0..12]: [278, 298, 299, 282, 298, 298, 299, 299, 299, 296, 299, 299]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851578116Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"[2]R generation[0..12]: [165, 174, 175, 166, 174, 174, 175, 175, 175, 172, 175, 175]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851592085Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851605943Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851619182Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.85163262Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851652578Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851677775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","Wait for a query_work_queue command to finish before sending IO pid":1891,"":"downstairs","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851707351Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851724829Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"0","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851739148Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"1","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851752586Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"client":"2","":"downstairs","session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851765974Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851778333Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"283bf56d-d3ce-4534-8031-3a49815e4fef is now active with session: 854808b7-19ea-4851-9a92-2cb68d900fbe","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851791162Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.85180326Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891,"session_id":"854808b7-19ea-4851-9a92-2cb68d900fbe"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:08.851815749Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 106 len 16384 data: 11 14 15 13 02/30 Write block 297 len 4096 data: 15 03/30 Read block 294 len 20480 04/30 Read block 266 len 40960 05/30 Read block 30 len 28672 06/30 Flush 07/30 Write block 461 len 8192 data: 16 17 08/30 Read block 450 len 16384 09/30 Read block 405 len 28672 10/30 Write block 456 len 20480 data: 13 14 14 17 16 11/30 Write block 435 len 4096 data: 12 12/30 Read block 440 len 32768 13/30 Read block 527 len 12288 14/30 Read block 541 len 24576 15/30 Read block 191 len 12288 16/30 Read block 211 len 24576 17/30 Write block 236 len 16384 data: 8 9 12 10 18/30 Read block 197 len 24576 19/30 Read block 442 len 4096 20/30 Write block 334 len 16384 data: 9 9 11 12 21/30 Read block 369 len 24576 22/30 Flush 23/30 Write block 583 len 24576 data: 14 10 11 9 13 12 24/30 Flush 25/30 Flush 26/30 Write block 366 len 4096 data: 12 27/30 Write block 199 len 24576 data: 14 17 15 15 15 17 28/30 Write block 10 len 40960 data: 12 13 14 14 15 16 19 18 20 22 29/30 Write block 0 len 36864 data: 4 4 5 6 8 7 10 14 12 30/30 Write block 70 len 20480 data: 15 13 10 11 13 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:08.870137291Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1891} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 278 278 278 T T T 1 020-039 174 174 174 298 298 298 F F F 2 040-059 175 175 175 299 299 299 F F F 3 060-079 166 166 166 282 282 282 T T T 4 080-099 174 174 174 298 298 298 F F F 5 100-119 176 176 176 300 300 300 F F F 6 120-139 175 175 175 299 299 299 F F F 7 140-159 175 175 175 299 299 299 F F F 8 160-179 175 175 175 299 299 299 F F F 9 180-199 172 172 172 296 296 296 T T T 10 200-219 175 175 175 299 299 299 T T T 11 220-239 176 176 176 301 301 301 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 170 170 170 291 291 291 F F F 14 280-299 176 176 176 300 300 300 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 176 176 176 301 301 301 F F F 17 340-359 175 175 175 299 299 299 F F F 18 360-379 175 175 175 299 299 299 T T T 19 380-399 175 175 175 299 299 299 F F F 20 400-419 175 175 175 299 299 299 F F F 21 420-439 176 176 176 301 301 301 F F F 22 440-459 176 176 176 301 301 301 F F F 23 460-479 176 176 176 301 301 301 F F F 24 480-499 171 171 171 293 293 293 F F F 25 500-519 175 175 175 299 299 299 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 171 171 171 293 293 293 F F F 28 560-579 174 174 174 298 298 298 F F F 29 580-599 176 176 176 302 302 302 F F F Max gen: 176, Max flush: 302 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 177 177 177 303 303 303 F F F 1 020-039 174 174 174 298 298 298 F F F 2 040-059 175 175 175 299 299 299 F F F 3 060-079 177 177 177 303 303 303 F F F 4 080-099 174 174 174 298 298 298 F F F 5 100-119 176 176 176 300 300 300 F F F 6 120-139 175 175 175 299 299 299 F F F 7 140-159 175 175 175 299 299 299 F F F 8 160-179 175 175 175 299 299 299 F F F 9 180-199 177 177 177 303 303 303 F F F 10 200-219 177 177 177 303 303 303 F F F 11 220-239 176 176 176 301 301 301 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 170 170 170 291 291 291 F F F 14 280-299 176 176 176 300 300 300 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 176 176 176 301 301 301 F F F 17 340-359 175 175 175 299 299 299 F F F 18 360-379 177 177 177 303 303 303 F F F 19 380-399 175 175 175 299 299 299 F F F 20 400-419 175 175 175 299 299 299 F F F 21 420-439 176 176 176 301 301 301 F F F 22 440-459 176 176 176 301 301 301 F F F 23 460-479 176 176 176 301 301 301 F F F 24 480-499 171 171 171 293 293 293 F F F 25 500-519 175 175 175 299 299 299 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 171 171 171 293 293 293 F F F 28 560-579 174 174 174 298 298 298 F F F 29 580-599 176 176 176 302 302 302 F F F Max gen: 177, Max flush: 303 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.145893529Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:13.189202455Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.189614837Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.221000915Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.221742608Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.221780694Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.221797662Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.2218168Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899} {"msg":"Crucible 57540410-1f08-4956-824f-c76e28dbca54 has session id: 76bdefb9-b136-4ba4-a344-4829607ff3ac","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.222338419Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Upstairs opts: Upstairs UUID: 57540410-1f08-4956-824f-c76e28dbca54, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.222387113Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Crucible stats registered with UUID: 57540410-1f08-4956-824f-c76e28dbca54","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.22241303Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.253002451Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.253071703Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899} {"msg":"57540410-1f08-4956-824f-c76e28dbca54 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.253090771Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.253107449Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"0","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.253161262Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"0","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.253197068Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"1","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.253233434Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"1","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.25326607Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"2","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.253297476Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"2","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.253323993Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.25335214Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.253378847Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:65258)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.253407474Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:34683)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.25343548Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:35693)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.253478285Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.254165215Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"1","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.25421187Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"2","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.254433284Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"0","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.254934885Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"1","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.25497675Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"2","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255009876Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"0","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255478442Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255601698Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"[0]R flush_numbers[0..12]: [303, 298, 299, 303, 298, 300, 299, 299, 299, 303, 303, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255676519Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"[0]R generation[0..12]: [177, 174, 175, 177, 174, 176, 175, 175, 175, 177, 177, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255696167Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255710665Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"[1]R flush_numbers[0..12]: [303, 298, 299, 303, 298, 300, 299, 299, 299, 303, 303, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255724663Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"[1]R generation[0..12]: [177, 174, 175, 177, 174, 176, 175, 175, 175, 177, 177, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255738622Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.25575264Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"[2]R flush_numbers[0..12]: [303, 298, 299, 303, 298, 300, 299, 299, 299, 303, 303, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255767868Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"[2]R generation[0..12]: [177, 174, 175, 177, 174, 176, 175, 175, 175, 177, 177, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255781806Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255795805Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255813333Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255827211Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255840419Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255853438Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"":"downstairs","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255866687Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255878705Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"0","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-01-13T15:58:13.255906052Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"1","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.25592136Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"client":"2","":"downstairs","session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255934919Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255947177Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"57540410-1f08-4956-824f-c76e28dbca54 is now active with session: 76bdefb9-b136-4ba4-a344-4829607ff3ac","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255959816Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255971874Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899,"session_id":"76bdefb9-b136-4ba4-a344-4829607ff3ac"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:13.255984203Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 346 len 12288 data: 18 17 16 02/30 Read block 138 len 20480 03/30 Read block 439 len 36864 04/30 Read block 499 len 24576 05/30 Read block 440 len 24576 06/30 Write block 122 len 36864 data: 14 17 17 15 17 19 17 13 9 07/30 Write block 61 len 8192 data: 9 9 08/30 Read block 501 len 20480 09/30 Write block 61 len 16384 data: 10 10 11 12 10/30 Flush 11/30 Write block 336 len 20480 data: 12 13 14 12 13 12/30 Read block 257 len 32768 13/30 Write block 380 len 8192 data: 11 12 14/30 Flush 15/30 Write block 460 len 20480 data: 17 17 18 16 12 16/30 Write block 499 len 24576 data: 16 19 18 18 18 15 17/30 Write block 366 len 24576 data: 13 11 11 9 9 10 18/30 Write block 397 len 24576 data: 17 16 18 17 18 20 19/30 Write block 57 len 16384 data: 12 11 12 9 20/30 Flush 21/30 Write block 209 len 4096 data: 16 22/30 Read block 112 len 24576 23/30 Read block 554 len 4096 24/30 Write block 410 len 32768 data: 19 19 20 20 17 17 15 15 25/30 Read block 118 len 32768 26/30 Read block 295 len 12288 27/30 Read block 582 len 16384 28/30 Read block 404 len 4096 29/30 Read block 502 len 24576 30/30 Read block 243 len 4096 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:13.274924183Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1899} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 177 177 177 303 303 303 F F F 1 020-039 174 174 174 298 298 298 F F F 2 040-059 178 178 178 306 306 306 F F F 3 060-079 178 178 178 306 306 306 F F F 4 080-099 174 174 174 298 298 298 F F F 5 100-119 176 176 176 300 300 300 F F F 6 120-139 178 178 178 304 304 304 F F F 7 140-159 175 175 175 299 299 299 F F F 8 160-179 175 175 175 299 299 299 F F F 9 180-199 177 177 177 303 303 303 F F F 10 200-219 177 177 177 303 303 303 T T T 11 220-239 176 176 176 301 301 301 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 170 170 170 291 291 291 F F F 14 280-299 176 176 176 300 300 300 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 178 178 178 305 305 305 F F F 17 340-359 178 178 178 305 305 305 F F F 18 360-379 178 178 178 306 306 306 F F F 19 380-399 178 178 178 306 306 306 F F F 20 400-419 178 178 178 306 306 306 T T T 21 420-439 176 176 176 301 301 301 F F F 22 440-459 176 176 176 301 301 301 F F F 23 460-479 178 178 178 306 306 306 F F F 24 480-499 178 178 178 306 306 306 F F F 25 500-519 178 178 178 306 306 306 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 171 171 171 293 293 293 F F F 28 560-579 174 174 174 298 298 298 F F F 29 580-599 176 176 176 302 302 302 F F F Max gen: 178, Max flush: 306 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 177 177 177 303 303 303 F F F 1 020-039 174 174 174 298 298 298 F F F 2 040-059 178 178 178 306 306 306 F F F 3 060-079 178 178 178 306 306 306 F F F 4 080-099 174 174 174 298 298 298 F F F 5 100-119 176 176 176 300 300 300 F F F 6 120-139 178 178 178 304 304 304 F F F 7 140-159 175 175 175 299 299 299 F F F 8 160-179 175 175 175 299 299 299 F F F 9 180-199 177 177 177 303 303 303 F F F 10 200-219 179 179 179 307 307 307 F F F 11 220-239 176 176 176 301 301 301 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 170 170 170 291 291 291 F F F 14 280-299 176 176 176 300 300 300 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 178 178 178 305 305 305 F F F 17 340-359 178 178 178 305 305 305 F F F 18 360-379 178 178 178 306 306 306 F F F 19 380-399 178 178 178 306 306 306 F F F 20 400-419 179 179 179 307 307 307 F F F 21 420-439 176 176 176 301 301 301 F F F 22 440-459 176 176 176 301 301 301 F F F 23 460-479 178 178 178 306 306 306 F F F 24 480-499 178 178 178 306 306 306 F F F 25 500-519 178 178 178 306 306 306 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 171 171 171 293 293 293 F F F 28 560-579 174 174 174 298 298 298 F F F 29 580-599 176 176 176 302 302 302 F F F Max gen: 179, Max flush: 307 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.572415324Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.617220536Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.617935302Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.617975797Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.617991406Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.618006824Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907} {"msg":"Crucible 469e6cb4-bd83-4635-ad90-a55824698c5d has session id: ff1a9662-5ecb-49f6-b290-9f226cefa515","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.61838595Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"Upstairs opts: Upstairs UUID: 469e6cb4-bd83-4635-ad90-a55824698c5d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.618431064Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"Crucible stats registered with UUID: 469e6cb4-bd83-4635-ad90-a55824698c5d","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.618460241Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.648430894Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.648530372Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907} {"msg":"469e6cb4-bd83-4635-ad90-a55824698c5d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.648563658Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.648596924Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"0","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.648644169Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"0","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.648677465Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"1","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.648784582Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"1","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.648827328Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"2","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.648862844Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"2","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.649171957Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.649216602Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.649260747Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"ds_connection connected from Ok(127.0.0.1:60078)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.649286154Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"ds_connection connected from Ok(127.0.0.1:65141)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.649302432Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"ds_connection connected from Ok(127.0.0.1:39426)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.64932109Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.649607746Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"0","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.649656731Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"1","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.649714304Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"2","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.650253531Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"1","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.650283548Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"2","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.650305245Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"0","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.650832014Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.650939001Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"[0]R flush_numbers[0..12]: [303, 298, 306, 306, 298, 300, 304, 299, 299, 303, 307, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651015372Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"[0]R generation[0..12]: [177, 174, 178, 178, 174, 176, 178, 175, 175, 177, 179, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.65103776Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651053368Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"[1]R flush_numbers[0..12]: [303, 298, 306, 306, 298, 300, 304, 299, 299, 303, 307, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651067696Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"[1]R generation[0..12]: [177, 174, 178, 178, 174, 176, 178, 175, 175, 177, 179, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651081774Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651100132Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"[2]R flush_numbers[0..12]: [303, 298, 306, 306, 298, 300, 304, 299, 299, 303, 307, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651115061Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"[2]R generation[0..12]: [177, 174, 178, 178, 174, 176, 178, 175, 175, 177, 179, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651129149Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651143147Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651156256Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651169654Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651182823Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651196001Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"":"downstairs","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.65120941Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651221798Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"0","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515Wait for a query_work_queue command to finish before sending IO "} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651244175Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"1","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651257574Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"client":"2","":"downstairs","session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651271032Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651283391Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"469e6cb4-bd83-4635-ad90-a55824698c5d is now active with session: ff1a9662-5ecb-49f6-b290-9f226cefa515","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651296469Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651310298Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907,"session_id":"ff1a9662-5ecb-49f6-b290-9f226cefa515"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:17.651322866Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 72 len 20480 data: 11 12 14 10 6 02/30 Write block 281 len 16384 data: 6 6 7 5 03/30 Read block 55 len 24576 04/30 Write block 396 len 12288 data: 10 18 17 05/30 Read block 335 len 24576 06/30 Write block 4 len 28672 data: 9 8 11 15 13 10 13 07/30 Flush 08/30 Write block 46 len 24576 data: 17 14 12 13 13 11 09/30 Read block 541 len 12288 10/30 Flush 11/30 Write block 370 len 16384 data: 10 11 10 12 12/30 Read block 75 len 28672 13/30 Read block 212 len 32768 14/30 Flush 15/30 Write block 292 len 20480 data: 19 18 19 15 12 16/30 Write block 427 len 12288 data: 15 14 14 17/30 Read block 268 len 24576 18/30 Read block 399 len 8192 19/30 Read block 362 len 40960 20/30 Write block 23 len 24576 data: 20 18 17 18 17 15 21/30 Write block 504 len 32768 data: 16 14 14 13 15 15 13 13 22/30 Read block 301 len 32768 23/30 Read block 56 len 32768 24/30 Flush 25/30 Write block 42 len 28672 data: 15 18 17 15 18 15 13 26/30 Flush 27/30 Write block 459 len 36864 data: 18 18 18 19 17 13 9 10 11 28/30 Write block 573 len 28672 data: 16 15 14 14 12 11 9 29/30 Write block 116 len 12288 data: 18 20 17 30/30 Write block 369 len 36864 data: 10 11 12 11 13 11 10 10 11 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:17.671578803Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1907} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 308 308 308 F F F 1 020-039 180 180 180 311 311 311 F F F 2 040-059 180 180 180 312 312 312 F F F 3 060-079 180 180 180 308 308 308 F F F 4 080-099 174 174 174 298 298 298 F F F 5 100-119 176 176 176 300 300 300 T T T 6 120-139 178 178 178 304 304 304 F F F 7 140-159 175 175 175 299 299 299 F F F 8 160-179 175 175 175 299 299 299 F F F 9 180-199 177 177 177 303 303 303 F F F 10 200-219 179 179 179 307 307 307 F F F 11 220-239 176 176 176 301 301 301 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 170 170 170 291 291 291 F F F 14 280-299 180 180 180 311 311 311 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 178 178 178 305 305 305 F F F 17 340-359 178 178 178 305 305 305 F F F 18 360-379 180 180 180 310 310 310 T T T 19 380-399 180 180 180 308 308 308 F F F 20 400-419 179 179 179 307 307 307 F F F 21 420-439 180 180 180 311 311 311 F F F 22 440-459 176 176 176 301 301 301 T T T 23 460-479 178 178 178 306 306 306 T T T 24 480-499 178 178 178 306 306 306 F F F 25 500-519 180 180 180 311 311 311 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 171 171 171 293 293 293 F F F 28 560-579 174 174 174 298 298 298 T T T 29 580-599 176 176 176 302 302 302 F F F Max gen: 180, Max flush: 312 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 308 308 308 F F F 1 020-039 180 180 180 311 311 311 F F F 2 040-059 180 180 180 312 312 312 F F F 3 060-079 180 180 180 308 308 308 F F F 4 080-099 174 174 174 298 298 298 F F F 5 100-119 181 181 181 313 313 313 F F F 6 120-139 178 178 178 304 304 304 F F F 7 140-159 175 175 175 299 299 299 F F F 8 160-179 175 175 175 299 299 299 F F F 9 180-199 177 177 177 303 303 303 F F F 10 200-219 179 179 179 307 307 307 F F F 11 220-239 176 176 176 301 301 301 F F F 12 240-259 162 162 162 274 274 274 F F F 13 260-279 170 170 170 291 291 291 F F F 14 280-299 180 180 180 311 311 311 F F F 15 300-319 165 165 165 278 278 278 F F F 16 320-339 178 178 178 305 305 305 F F F 17 340-359 178 178 178 305 305 305 F F F 18 360-379 181 181 181 313 313 313 F F F 19 380-399 180 180 180 308 308 308 F F F 20 400-419 179 179 179 307 307 307 F F F 21 420-439 180 180 180 311 311 311 F F F 22 440-459 181 181 181 313 313 313 F F F 23 460-479 181 181 181 313 313 313 F F F 24 480-499 178 178 178 306 306 306 F F F 25 500-519 180 180 180 311 311 311 F F F 26 520-539 175 175 175 299 299 299 F F F 27 540-559 171 171 171 293 293 293 F F F 28 560-579 181 181 181 313 313 313 F F F 29 580-599 176 176 176 302 302 302 F F F Max gen: 181, Max flush: 313 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:21.956134234Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:21.999587614Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.000337936Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.000377711Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.000396739Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.000412387Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915} {"msg":"Crucible f69d2bcb-0400-462c-bc82-73ea2bbf90e4 has session id: a88d21fe-b5e9-4623-b393-7be913a35aae","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.000824759Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"Upstairs opts: Upstairs UUID: f69d2bcb-0400-462c-bc82-73ea2bbf90e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.000870984Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"Crucible stats registered with UUID: f69d2bcb-0400-462c-bc82-73ea2bbf90e4","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.000895481Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032152514Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032206248Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915} {"msg":"f69d2bcb-0400-462c-bc82-73ea2bbf90e4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032222146Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032238264Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"0","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032266621Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"0","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032284308Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"1","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032302137Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"1","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032318795Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"2","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032336532Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"2","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032353351Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032377278Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032394606Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"ds_connection connected from Ok(127.0.0.1:53344)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032751514Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"ds_connection connected from Ok(127.0.0.1:56293)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032797948Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"ds_connection connected from Ok(127.0.0.1:64046)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.032818666Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.033421646Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"0","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.033462201Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"2","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.0335553Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"1","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034147421Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"2","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034177208Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"0","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034195916Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"1","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034773058Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034799685Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"[0]R flush_numbers[0..12]: [308, 311, 312, 308, 298, 313, 304, 299, 299, 303, 307, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034866177Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"[0]R generation[0..12]: [180, 180, 180, 180, 174, 181, 178, 175, 175, 177, 179, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034891964Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034906933Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"[1]R flush_numbers[0..12]: [308, 311, 312, 308, 298, 313, 304, 299, 299, 303, 307, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034921121Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"[1]R generation[0..12]: [180, 180, 180, 180, 174, 181, 178, 175, 175, 177, 179, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034934959Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034949057Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"[2]R flush_numbers[0..12]: [308, 311, 312, 308, 298, 313, 304, 299, 299, 303, 307, 301]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034963186Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"[2]R generation[0..12]: [180, 180, 180, 180, 174, 181, 178, 175, 175, 177, 179, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034977154Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.034991283Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.035004451Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.03501778Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.035031128Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.035045296Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"":"downstairs","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.035058905Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.035071023Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"0","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"}Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.035093581Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"1","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.035108809Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"client":"2","":"downstairs","session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.035122417Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.035135006Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"f69d2bcb-0400-462c-bc82-73ea2bbf90e4 is now active with session: a88d21fe-b5e9-4623-b393-7be913a35aae","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.035147784Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.035159893Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915,"session_id":"a88d21fe-b5e9-4623-b393-7be913a35aae"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:22.035172451Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 377 len 16384 data: 12 10 10 12 02/30 Read block 334 len 16384 03/30 Read block 379 len 28672 04/30 Flush 05/30 Write block 316 len 12288 data: 8 8 9 06/30 Read block 466 len 28672 07/30 Read block 33 len 40960 08/30 Write block 153 len 4096 data: 16 09/30 Read block 438 len 40960 10/30 Write block 564 len 40960 data: 16 15 16 14 14 16 15 13 19 17 11/30 Write block 452 len 12288 data: 13 13 12 12/30 Read block 15 len 32768 13/30 Write block 374 len 32768 data: 12 11 11 13 11 11 13 13 14/30 Flush 15/30 Write block 281 len 24576 data: 7 7 8 6 7 9 16/30 Read block 351 len 12288 17/30 Write block 387 len 4096 data: 9 18/30 Read block 434 len 24576 19/30 Write block 10 len 16384 data: 14 14 15 15 20/30 Write block 545 len 32768 data: 15 17 14 12 11 10 13 11 21/30 Write block 206 len 16384 data: 16 17 14 17 22/30 Write block 173 len 28672 data: 8 9 10 12 14 15 15 23/30 Read block 219 len 24576 24/30 Flush 25/30 Write block 526 len 12288 data: 10 10 11 26/30 Read block 582 len 20480 27/30 Write block 233 len 36864 data: 12 12 11 9 10 13 11 9 10 28/30 Write block 254 len 36864 data: 15 17 15 16 17 17 17 17 19 29/30 Write block 509 len 40960 data: 16 14 14 14 12 15 13 15 13 13 30/30 Read block 196 len 28672 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 6 5 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:22.055630194Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1915} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 316 316 316 F F F 1 020-039 180 180 180 311 311 311 F F F 2 040-059 180 180 180 312 312 312 F F F 3 060-079 180 180 180 308 308 308 F F F 4 080-099 174 174 174 298 298 298 F F F 5 100-119 181 181 181 313 313 313 F F F 6 120-139 178 178 178 304 304 304 F F F 7 140-159 182 182 182 315 315 315 F F F 8 160-179 182 182 182 316 316 316 F F F 9 180-199 177 177 177 303 303 303 F F F 10 200-219 182 182 182 316 316 316 F F F 11 220-239 176 176 176 301 301 301 T T T 12 240-259 162 162 162 274 274 274 T T T 13 260-279 170 170 170 291 291 291 T T T 14 280-299 182 182 182 316 316 316 F F F 15 300-319 182 182 182 315 315 315 F F F 16 320-339 178 178 178 305 305 305 F F F 17 340-359 178 178 178 305 305 305 F F F 18 360-379 182 182 182 315 315 315 F F F 19 380-399 182 182 182 316 316 316 F F F 20 400-419 179 179 179 307 307 307 F F F 21 420-439 180 180 180 311 311 311 F F F 22 440-459 182 182 182 315 315 315 F F F 23 460-479 181 181 181 313 313 313 F F F 24 480-499 178 178 178 306 306 306 F F F 25 500-519 180 180 180 311 311 311 T T T 26 520-539 175 175 175 299 299 299 T T T 27 540-559 182 182 182 316 316 316 F F F 28 560-579 182 182 182 315 315 315 F F F 29 580-599 176 176 176 302 302 302 F F F Max gen: 182, Max flush: 316 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 316 316 316 F F F 1 020-039 180 180 180 311 311 311 F F F 2 040-059 180 180 180 312 312 312 F F F 3 060-079 180 180 180 308 308 308 F F F 4 080-099 174 174 174 298 298 298 F F F 5 100-119 181 181 181 313 313 313 F F F 6 120-139 178 178 178 304 304 304 F F F 7 140-159 182 182 182 315 315 315 F F F 8 160-179 182 182 182 316 316 316 F F F 9 180-199 177 177 177 303 303 303 F F F 10 200-219 182 182 182 316 316 316 F F F 11 220-239 183 183 183 317 317 317 F F F 12 240-259 183 183 183 317 317 317 F F F 13 260-279 183 183 183 317 317 317 F F F 14 280-299 182 182 182 316 316 316 F F F 15 300-319 182 182 182 315 315 315 F F F 16 320-339 178 178 178 305 305 305 F F F 17 340-359 178 178 178 305 305 305 F F F 18 360-379 182 182 182 315 315 315 F F F 19 380-399 182 182 182 316 316 316 F F F 20 400-419 179 179 179 307 307 307 F F F 21 420-439 180 180 180 311 311 311 F F F 22 440-459 182 182 182 315 315 315 F F F 23 460-479 181 181 181 313 313 313 F F F 24 480-499 178 178 178 306 306 306 F F F 25 500-519 183 183 183 317 317 317 F F F 26 520-539 183 183 183 317 317 317 F F F 27 540-559 182 182 182 316 316 316 F F F 28 560-579 182 182 182 315 315 315 F F F 29 580-599 176 176 176 302 302 302 F F F Max gen: 183, Max flush: 317 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.331523505Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.375915165Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.376696654Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.376760467Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.376792613Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.376824369Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923} {"msg":"Crucible 5d877875-c122-4733-8d44-bd7db300b165 has session id: c152a8c2-067b-4197-bc7c-997ce8fa70b1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.377227832Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Upstairs opts: Upstairs UUID: 5d877875-c122-4733-8d44-bd7db300b165, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.377260778Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Crucible stats registered with UUID: 5d877875-c122-4733-8d44-bd7db300b165","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.377276317Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.407677629Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.407723654Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923} {"msg":"5d877875-c122-4733-8d44-bd7db300b165 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.407847269Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.407876276Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"0","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.407901163Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"0","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.407916761Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"1","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.407935549Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"1","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.407959866Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"2","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.407988213Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"2","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.40801416Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.408041657Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.408068573Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51211)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.40809766Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:61077)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.408126757Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"ds_connection connected from Ok(127.0.0.1:64385)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.408155923Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.408893377Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"0","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.408925504Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"1","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.409068497Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"2","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.409502796Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"0","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.409595795Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"1","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.40989637Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"2","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410045183Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410115945Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"[0]R flush_numbers[0..12]: [316, 311, 312, 308, 298, 313, 304, 315, 316, 303, 316, 317]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410436407Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"[0]R generation[0..12]: [182, 180, 180, 180, 174, 181, 178, 182, 182, 177, 182, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410458075Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410473153Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"[1]R flush_numbers[0..12]: [316, 311, 312, 308, 298, 313, 304, 315, 316, 303, 316, 317]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410487021Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"[1]R generation[0..12]: [182, 180, 180, 180, 174, 181, 178, 182, 182, 177, 182, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.41050092Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410515308Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"[2]R flush_numbers[0..12]: [316, 311, 312, 308, 298, 313, 304, 315, 316, 303, 316, 317]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410529326Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"[2]R generation[0..12]: [182, 180, 180, 180, 174, 181, 178, 182, 182, 177, 182, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410543305Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410557423Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410570552Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.41058394Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Next flush: 318","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-01-13T15:58:26.410600948Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410621746Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"":"downstairs","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410635214Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410647403Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"0","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410660811Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"1","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.41067406Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"client":"2","":"downstairs","session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410687448Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410699726Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"5d877875-c122-4733-8d44-bd7db300b165 is now active with session: c152a8c2-067b-4197-bc7c-997ce8fa70b1","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410712535Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410724484Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923,"session_id":"c152a8c2-067b-4197-bc7c-997ce8fa70b1"} {"msg":"The guest has finished waiting for activation with:184"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:26.410740012Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 126 len 20480 data: 18 20 18 14 10 02/30 Read block 21 len 24576 03/30 Read block 468 len 28672 04/30 Write block 414 len 12288 data: 18 18 16 05/30 Write block 268 len 8192 data: 11 10 06/30 Read block 420 len 28672 07/30 Read block 229 len 16384 08/30 Write block 448 len 40960 data: 11 14 13 14 14 14 13 14 14 15 09/30 Write block 154 len 36864 data: 14 15 14 13 15 16 18 19 23 10/30 Read block 238 len 28672 11/30 Read block 346 len 16384 12/30 Write block 420 len 40960 data: 18 19 17 17 16 15 17 16 15 15 13/30 Write block 459 len 32768 data: 19 19 19 20 18 14 10 11 14/30 Read block 246 len 16384 15/30 Read block 45 len 8192 16/30 Write block 239 len 24576 data: 12 10 11 12 8 7 17/30 Read block 53 len 32768 18/30 Write block 544 len 12288 data: 16 16 18 19/30 Read block 156 len 4096 20/30 Write block 450 len 24576 data: 14 15 15 15 14 15 21/30 Read block 35 len 32768 22/30 Write block 520 len 36864 data: 13 14 14 14 13 10 11 11 12 23/30 Write block 451 len 12288 data: 16 16 16 24/30 Write block 420 len 4096 data: 19 25/30 Write block 215 len 16384 data: 18 17 16 19 26/30 Read block 588 len 32768 27/30 Read block 23 len 24576 28/30 Write block 411 len 32768 data: 20 21 21 19 19 17 16 14 29/30 Write block 533 len 12288 data: 16 16 14 30/30 Write block 88 len 28672 data: 16 21 18 20 18 22 19 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:26.43020739Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1923} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 316 316 316 F F F 1 020-039 180 180 180 311 311 311 F F F 2 040-059 180 180 180 312 312 312 F F F 3 060-079 180 180 180 308 308 308 F F F 4 080-099 174 174 174 298 298 298 T T T 5 100-119 181 181 181 313 313 313 F F F 6 120-139 178 178 178 304 304 304 T T T 7 140-159 182 182 182 315 315 315 T T T 8 160-179 182 182 182 316 316 316 T T T 9 180-199 177 177 177 303 303 303 F F F 10 200-219 182 182 182 316 316 316 T T T 11 220-239 183 183 183 317 317 317 T T T 12 240-259 183 183 183 317 317 317 T T T 13 260-279 183 183 183 317 317 317 T T T 14 280-299 182 182 182 316 316 316 F F F 15 300-319 182 182 182 315 315 315 F F F 16 320-339 178 178 178 305 305 305 F F F 17 340-359 178 178 178 305 305 305 F F F 18 360-379 182 182 182 315 315 315 F F F 19 380-399 182 182 182 316 316 316 F F F 20 400-419 179 179 179 307 307 307 T T T 21 420-439 180 180 180 311 311 311 T T T 22 440-459 182 182 182 315 315 315 T T T 23 460-479 181 181 181 313 313 313 T T T 24 480-499 178 178 178 306 306 306 F F F 25 500-519 183 183 183 317 317 317 F F F 26 520-539 183 183 183 317 317 317 T T T 27 540-559 182 182 182 316 316 316 T T T 28 560-579 182 182 182 315 315 315 F F F 29 580-599 176 176 176 302 302 302 F F F Max gen: 183, Max flush: 317 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 316 316 316 F F F 1 020-039 180 180 180 311 311 311 F F F 2 040-059 180 180 180 312 312 312 F F F 3 060-079 180 180 180 308 308 308 F F F 4 080-099 184 184 184 318 318 318 F F F 5 100-119 181 181 181 313 313 313 F F F 6 120-139 184 184 184 318 318 318 F F F 7 140-159 184 184 184 318 318 318 F F F 8 160-179 184 184 184 318 318 318 F F F 9 180-199 177 177 177 303 303 303 F F F 10 200-219 184 184 184 318 318 318 F F F 11 220-239 184 184 184 318 318 318 F F F 12 240-259 184 184 184 318 318 318 F F F 13 260-279 184 184 184 318 318 318 F F F 14 280-299 182 182 182 316 316 316 F F F 15 300-319 182 182 182 315 315 315 F F F 16 320-339 178 178 178 305 305 305 F F F 17 340-359 178 178 178 305 305 305 F F F 18 360-379 182 182 182 315 315 315 F F F 19 380-399 182 182 182 316 316 316 F F F 20 400-419 184 184 184 318 318 318 F F F 21 420-439 184 184 184 318 318 318 F F F 22 440-459 184 184 184 318 318 318 F F F 23 460-479 184 184 184 318 318 318 F F F 24 480-499 178 178 178 306 306 306 F F F 25 500-519 183 183 183 317 317 317 F F F 26 520-539 184 184 184 318 318 318 F F F 27 540-559 184 184 184 318 318 318 F F F 28 560-579 182 182 182 315 315 315 F F F 29 580-599 176 176 176 302 302 302 F F F Max gen: 184, Max flush: 318 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.761956683Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:30.805528948Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.80593685Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.836951461Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.837550541Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.837594426Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.837617233Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.837635811Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931} {"msg":"Crucible fcf90346-f482-4e1e-a954-e976df8e50df has session id: 54b7e142-5e02-4ea8-8e8a-44e2de171325","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.838091808Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Upstairs opts: Upstairs UUID: fcf90346-f482-4e1e-a954-e976df8e50df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.838138442Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Crucible stats registered with UUID: fcf90346-f482-4e1e-a954-e976df8e50df","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.83816395Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868508249Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868564562Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931} {"msg":"fcf90346-f482-4e1e-a954-e976df8e50df active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868595089Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868624145Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"0","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.86867323Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"0","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868711155Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"1","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868746601Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"1","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868780267Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"2","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868815773Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"2","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868852249Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868884265Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868915871Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"ds_connection connected from Ok(127.0.0.1:34435)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868948937Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"ds_connection connected from Ok(127.0.0.1:35104)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.868980204Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"ds_connection connected from Ok(127.0.0.1:33403)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.8690095Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.86978582Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"1","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.869816176Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"2","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.869887248Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"0","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.870482488Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"1","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.870521584Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"2","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.870549531Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"0","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871139652Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871168259Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"[0]R flush_numbers[0..12]: [316, 311, 312, 308, 318, 313, 318, 318, 318, 303, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871189746Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"[0]R generation[0..12]: [182, 180, 180, 180, 184, 181, 184, 184, 184, 177, 184, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871209584Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871224732Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"[1]R flush_numbers[0..12]: [316, 311, 312, 308, 318, 313, 318, 318, 318, 303, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.87123919Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"[1]R generation[0..12]: [182, 180, 180, 180, 184, 181, 184, 184, 184, 177, 184, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871253438Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871267657Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"[2]R flush_numbers[0..12]: [316, 311, 312, 308, 318, 313, 318, 318, 318, 303, 318, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871281535Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"[2]R generation[0..12]: [182, 180, 180, 180, 184, 181, 184, 184, 184, 177, 184, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871295224Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871309182Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Max found gen is 185","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.87132229Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Generation requested: 186 >= found:185","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871335859Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871348907Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871362896Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"":"downstairs","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871376284Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871394922Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"0","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.87141136Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"1","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871424949Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"client":"2","":"downstairs","session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871438297Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871450655Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"fcf90346-f482-4e1e-a954-e976df8e50df is now active with session: 54b7e142-5e02-4ea8-8e8a-44e2de171325","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871463364Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871475452Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931,"session_id":"54b7e142-5e02-4ea8-8e8a-44e2de171325"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:30.871487751Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 42 len 4096 data: 16 02/30 Flush 03/30 Write block 485 len 12288 data: 12 13 14 04/30 Flush 05/30 Write block 563 len 36864 data: 16 17 16 17 15 15 17 16 14 06/30 Write block 184 len 36864 data: 14 12 10 8 12 15 14 16 12 07/30 Read block 472 len 28672 08/30 Read block 448 len 8192 09/30 Write block 593 len 28672 data: 16 18 15 15 9 8 6 10/30 Write block 233 len 12288 data: 13 13 12 11/30 Write block 435 len 24576 data: 13 12 15 14 11 10 12/30 Write block 103 len 40960 data: 12 12 14 12 15 16 14 11 10 11 13/30 Write block 327 len 24576 data: 15 17 14 13 11 11 14/30 Read block 412 len 24576 15/30 Read block 361 len 32768 16/30 Write block 199 len 16384 data: 15 18 16 16 17/30 Write block 513 len 4096 data: 13 18/30 Write block 288 len 4096 data: 10 19/30 Read block 134 len 40960 20/30 Write block 62 len 28672 data: 11 12 13 11 15 18 17 21/30 Read block 446 len 24576 22/30 Read block 267 len 36864 23/30 Read block 573 len 36864 24/30 Write block 571 len 8192 data: 15 20 25/30 Read block 68 len 8192 26/30 Read block 196 len 4096 27/30 Read block 545 len 36864 28/30 Read block 200 len 4096 29/30 Write block 581 len 36864 data: 11 12 15 11 12 10 14 13 12 30/30 Write block 430 len 24576 data: 15 12 15 13 12 14 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 9 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:30.889989202Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1931} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 316 316 316 F F F 1 020-039 180 180 180 311 311 311 F F F 2 040-059 186 186 186 319 319 319 F F F 3 060-079 180 180 180 308 308 308 T T T 4 080-099 184 184 184 318 318 318 F F F 5 100-119 181 181 181 313 313 313 T T T 6 120-139 184 184 184 318 318 318 F F F 7 140-159 184 184 184 318 318 318 F F F 8 160-179 184 184 184 318 318 318 F F F 9 180-199 177 177 177 303 303 303 T T T 10 200-219 184 184 184 318 318 318 T T T 11 220-239 184 184 184 318 318 318 T T T 12 240-259 184 184 184 318 318 318 F F F 13 260-279 184 184 184 318 318 318 F F F 14 280-299 182 182 182 316 316 316 T T T 15 300-319 182 182 182 315 315 315 F F F 16 320-339 178 178 178 305 305 305 T T T 17 340-359 178 178 178 305 305 305 F F F 18 360-379 182 182 182 315 315 315 F F F 19 380-399 182 182 182 316 316 316 F F F 20 400-419 184 184 184 318 318 318 F F F 21 420-439 184 184 184 318 318 318 T T T 22 440-459 184 184 184 318 318 318 T T T 23 460-479 184 184 184 318 318 318 F F F 24 480-499 186 186 186 320 320 320 F F F 25 500-519 183 183 183 317 317 317 T T T 26 520-539 184 184 184 318 318 318 F F F 27 540-559 184 184 184 318 318 318 F F F 28 560-579 182 182 182 315 315 315 T T T 29 580-599 176 176 176 302 302 302 T T T Max gen: 186, Max flush: 320 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 316 316 316 F F F 1 020-039 180 180 180 311 311 311 F F F 2 040-059 186 186 186 319 319 319 F F F 3 060-079 187 187 187 321 321 321 F F F 4 080-099 184 184 184 318 318 318 F F F 5 100-119 187 187 187 321 321 321 F F F 6 120-139 184 184 184 318 318 318 F F F 7 140-159 184 184 184 318 318 318 F F F 8 160-179 184 184 184 318 318 318 F F F 9 180-199 187 187 187 321 321 321 F F F 10 200-219 187 187 187 321 321 321 F F F 11 220-239 187 187 187 321 321 321 F F F 12 240-259 184 184 184 318 318 318 F F F 13 260-279 184 184 184 318 318 318 F F F 14 280-299 187 187 187 321 321 321 F F F 15 300-319 182 182 182 315 315 315 F F F 16 320-339 187 187 187 321 321 321 F F F 17 340-359 178 178 178 305 305 305 F F F 18 360-379 182 182 182 315 315 315 F F F 19 380-399 182 182 182 316 316 316 F F F 20 400-419 184 184 184 318 318 318 F F F 21 420-439 187 187 187 321 321 321 F F F 22 440-459 187 187 187 321 321 321 F F F 23 460-479 184 184 184 318 318 318 F F F 24 480-499 186 186 186 320 320 320 F F F 25 500-519 187 187 187 321 321 321 F F F 26 520-539 184 184 184 318 318 318 F F F 27 540-559 184 184 184 318 318 318 F F F 28 560-579 187 187 187 321 321 321 F F F 29 580-599 187 187 187 321 321 321 F F F Max gen: 187, Max flush: 321 Begin loop 94 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.242472167Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.287553266Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.288449882Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.288497026Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.288514514Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.288532882Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939} {"msg":"Crucible fc678722-3450-44d3-949e-ef921c99bde6 has session id: dc5cab50-d746-4927-984c-2b79ac84bd2e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.289006817Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Upstairs opts: Upstairs UUID: fc678722-3450-44d3-949e-ef921c99bde6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.289054211Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Crucible stats registered with UUID: fc678722-3450-44d3-949e-ef921c99bde6","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.289079068Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.319959265Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320052324Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939} {"msg":"fc678722-3450-44d3-949e-ef921c99bde6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320073812Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320096839Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"0","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320147413Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"0","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320186138Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"1","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320228074Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"1","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.32026081Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"2","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320291966Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"2","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320324502Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320362038Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320405023Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33550)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320441958Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61667)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320694009Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49128)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.320727335Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.32119806Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"2","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.321232516Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"0","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.321494596Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"1","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322078287Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"2","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322118593Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"0","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322153869Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"1","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.32266139Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322796924Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"[0]R flush_numbers[0..12]: [316, 311, 319, 321, 318, 321, 318, 318, 318, 321, 321, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322849988Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"[0]R generation[0..12]: [182, 180, 186, 187, 184, 187, 184, 184, 184, 187, 187, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322870805Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322886193Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"[1]R flush_numbers[0..12]: [316, 311, 319, 321, 318, 321, 318, 318, 318, 321, 321, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322900402Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"[1]R generation[0..12]: [182, 180, 186, 187, 184, 187, 184, 184, 184, 187, 187, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.32291433Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322928568Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"[2]R flush_numbers[0..12]: [316, 311, 319, 321, 318, 321, 318, 318, 318, 321, 321, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322942367Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"[2]R generation[0..12]: [182, 180, 186, 187, 184, 187, 184, 184, 184, 187, 187, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322960525Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322974883Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.322988181Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.3230016Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.323014638Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.323027827Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.323055853Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.323080101Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"0","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.323106178Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"1","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.323132025Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"client":"2","":"downstairs","session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.323156692Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.323177869Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"fc678722-3450-44d3-949e-ef921c99bde6 is now active with session: dc5cab50-d746-4927-984c-2b79ac84bd2e","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.323192508Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.323204636Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939,"session_id":"dc5cab50-d746-4927-984c-2b79ac84bd2e"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:35.323217035Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 155 len 28672 data: 16 15 14 16 17 19 20 02/30 Read block 321 len 8192 03/30 Write block 145 len 16384 data: 13 11 12 17 04/30 Flush 05/30 Write block 329 len 4096 data: 15 06/30 Write block 454 len 20480 data: 15 16 15 16 15 07/30 Read block 316 len 32768 08/30 Write block 136 len 24576 data: 13 12 12 11 16 14 09/30 Read block 311 len 40960 10/30 Write block 341 len 40960 data: 15 18 16 16 16 19 18 17 13 14 11/30 Write block 283 len 20480 data: 9 7 8 10 10 12/30 Read block 464 len 16384 13/30 Read block 384 len 32768 14/30 Read block 342 len 12288 15/30 Read block 488 len 24576 16/30 Flush 17/30 Write block 246 len 32768 data: 8 7 10 8 10 9 11 14 18/30 Flush 19/30 Flush 20/30 Write block 498 len 32768 data: 14 17 20 19 19 19 17 15 21/30 Read block 296 len 12288 22/30 Write block 513 len 8192 data: 14 16 23/30 Write block 55 len 24576 data: 16 12 13 12 13 10 24/30 Write block 435 len 32768 data: 15 13 16 15 12 11 9 9 25/30 Write block 458 len 28672 data: 16 20 20 20 21 19 15 26/30 Flush 27/30 Write block 513 len 24576 data: 15 17 14 16 14 14 28/30 Read block 408 len 28672 29/30 Write block 254 len 24576 data: 16 18 16 17 18 18 30/30 Write block 11 len 36864 data: 15 16 16 16 17 20 19 21 23 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:35.343948226Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1939} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 316 316 316 T T T 1 020-039 180 180 180 311 311 311 F F F 2 040-059 188 188 188 326 326 326 F F F 3 060-079 188 188 188 326 326 326 F F F 4 080-099 184 184 184 318 318 318 F F F 5 100-119 187 187 187 321 321 321 F F F 6 120-139 188 188 188 323 323 323 F F F 7 140-159 188 188 188 323 323 323 F F F 8 160-179 188 188 188 322 322 322 F F F 9 180-199 187 187 187 321 321 321 F F F 10 200-219 187 187 187 321 321 321 F F F 11 220-239 187 187 187 321 321 321 F F F 12 240-259 188 188 188 324 324 324 T T T 13 260-279 184 184 184 318 318 318 F F F 14 280-299 188 188 188 323 323 323 F F F 15 300-319 182 182 182 315 315 315 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 188 188 188 323 323 323 F F F 18 360-379 182 182 182 315 315 315 F F F 19 380-399 182 182 182 316 316 316 F F F 20 400-419 184 184 184 318 318 318 F F F 21 420-439 188 188 188 326 326 326 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 188 188 188 326 326 326 F F F 24 480-499 188 188 188 326 326 326 F F F 25 500-519 188 188 188 326 326 326 T T T 26 520-539 184 184 184 318 318 318 F F F 27 540-559 184 184 184 318 318 318 F F F 28 560-579 187 187 187 321 321 321 F F F 29 580-599 187 187 187 321 321 321 F F F Max gen: 188, Max flush: 326 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 327 327 327 F F F 1 020-039 180 180 180 311 311 311 F F F 2 040-059 188 188 188 326 326 326 F F F 3 060-079 188 188 188 326 326 326 F F F 4 080-099 184 184 184 318 318 318 F F F 5 100-119 187 187 187 321 321 321 F F F 6 120-139 188 188 188 323 323 323 F F F 7 140-159 188 188 188 323 323 323 F F F 8 160-179 188 188 188 322 322 322 F F F 9 180-199 187 187 187 321 321 321 F F F 10 200-219 187 187 187 321 321 321 F F F 11 220-239 187 187 187 321 321 321 F F F 12 240-259 189 189 189 327 327 327 F F F 13 260-279 184 184 184 318 318 318 F F F 14 280-299 188 188 188 323 323 323 F F F 15 300-319 182 182 182 315 315 315 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 188 188 188 323 323 323 F F F 18 360-379 182 182 182 315 315 315 F F F 19 380-399 182 182 182 316 316 316 F F F 20 400-419 184 184 184 318 318 318 F F F 21 420-439 188 188 188 326 326 326 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 188 188 188 326 326 326 F F F 24 480-499 188 188 188 326 326 326 F F F 25 500-519 189 189 189 327 327 327 F F F 26 520-539 184 184 184 318 318 318 F F F 27 540-559 184 184 184 318 318 318 F F F 28 560-579 187 187 187 321 321 321 F F F 29 580-599 187 187 187 321 321 321 F F F Max gen: 189, Max flush: 327 Begin loop 95 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.616433005Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:39.658950623Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.659423968Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.690355929Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.690967897Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.691006163Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.691020781Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.691040469Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947} {"msg":"Crucible 42f6ed0b-af59-4a41-8030-728f479aea4f has session id: c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.69145791Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Upstairs opts: Upstairs UUID: 42f6ed0b-af59-4a41-8030-728f479aea4f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.691506134Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Crucible stats registered with UUID: 42f6ed0b-af59-4a41-8030-728f479aea4f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.691530802Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.721398086Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.721439292Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947} {"msg":"42f6ed0b-af59-4a41-8030-728f479aea4f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.721471228Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.721489036Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"0","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.721519472Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"0","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.72153842Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"1","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.721556388Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"1","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.721573076Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"2","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.721597943Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"2","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.72162975Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.721664506Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.721683943Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:45311)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.722104284Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:35119)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.722157038Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:35670)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.722192054Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.722751839Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"1","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.722785435Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"2","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.722975513Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"0","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.723524538Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"1","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.723554775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"0","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.723581452Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"2","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724122409Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724149736Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"[0]R flush_numbers[0..12]: [327, 311, 326, 326, 318, 321, 323, 323, 322, 321, 321, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724217297Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"[0]R generation[0..12]: [189, 180, 188, 188, 184, 187, 188, 188, 188, 187, 187, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724237435Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724258303Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"[1]R flush_numbers[0..12]: [327, 311, 326, 326, 318, 321, 323, 323, 322, 321, 321, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724273401Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"[1]R generation[0..12]: [189, 180, 188, 188, 184, 187, 188, 188, 188, 187, 187, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724287429Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724301578Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"[2]R flush_numbers[0..12]: [327, 311, 326, 326, 318, 321, 323, 323, 322, 321, 321, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724315686Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"[2]R generation[0..12]: [189, 180, 188, 188, 184, 187, 188, 188, 188, 187, 187, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724329775Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724343863Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724357181Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.72437054Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724384618Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724404726Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"":"downstairs","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724417954Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724430073Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"0","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724443571Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"1","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724458549Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"client":"2","":"downstairs","session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724472078Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724484656Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"42f6ed0b-af59-4a41-8030-728f479aea4f is now active with session: c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724497695Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724509914Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947,"session_id":"c3eb7a79-ae7f-4ff2-afae-d1b6529b08f9"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:39.724522282Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 256 len 8192 data: 17 18 02/30 Read block 260 len 12288 03/30 Flush 04/30 Write block 515 len 12288 data: 15 17 15 05/30 Read block 536 len 16384 06/30 Read block 258 len 32768 07/30 Read block 372 len 8192 08/30 Read block 467 len 20480 09/30 Flush 10/30 Write block 416 len 28672 data: 18 17 15 18 20 20 18 11/30 Write block 364 len 32768 data: 14 14 14 12 12 11 12 13 12/30 Read block 445 len 24576 13/30 Read block 59 len 16384 14/30 Read block 186 len 4096 15/30 Write block 379 len 40960 data: 12 14 14 13 12 13 11 11 10 9 16/30 Read block 481 len 40960 17/30 Read block 449 len 36864 18/30 Write block 142 len 12288 data: 12 13 16 19/30 Read block 39 len 20480 20/30 Write block 78 len 32768 data: 7 6 7 7 13 11 9 11 21/30 Write block 262 len 24576 data: 20 18 15 14 16 15 22/30 Write block 348 len 32768 data: 18 14 15 12 11 8 13 11 23/30 Read block 97 len 32768 24/30 Read block 125 len 40960 25/30 Write block 489 len 8192 data: 15 12 26/30 Write block 9 len 8192 data: 11 15 27/30 Read block 419 len 8192 28/30 Read block 132 len 40960 29/30 Read block 269 len 8192 30/30 Read block 360 len 16384 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 7 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:39.743942976Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1947} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 327 327 327 T T T 1 020-039 180 180 180 311 311 311 F F F 2 040-059 188 188 188 326 326 326 F F F 3 060-079 188 188 188 326 326 326 T T T 4 080-099 184 184 184 318 318 318 T T T 5 100-119 187 187 187 321 321 321 F F F 6 120-139 188 188 188 323 323 323 F F F 7 140-159 188 188 188 323 323 323 T T T 8 160-179 188 188 188 322 322 322 F F F 9 180-199 187 187 187 321 321 321 F F F 10 200-219 187 187 187 321 321 321 F F F 11 220-239 187 187 187 321 321 321 F F F 12 240-259 190 190 190 328 328 328 F F F 13 260-279 184 184 184 318 318 318 T T T 14 280-299 188 188 188 323 323 323 F F F 15 300-319 182 182 182 315 315 315 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 188 188 188 323 323 323 T T T 18 360-379 182 182 182 315 315 315 T T T 19 380-399 182 182 182 316 316 316 T T T 20 400-419 184 184 184 318 318 318 T T T 21 420-439 188 188 188 326 326 326 T T T 22 440-459 188 188 188 326 326 326 F F F 23 460-479 188 188 188 326 326 326 F F F 24 480-499 188 188 188 326 326 326 T T T 25 500-519 190 190 190 329 329 329 F F F 26 520-539 184 184 184 318 318 318 F F F 27 540-559 184 184 184 318 318 318 F F F 28 560-579 187 187 187 321 321 321 F F F 29 580-599 187 187 187 321 321 321 F F F Max gen: 190, Max flush: 329 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 330 330 330 F F F 1 020-039 180 180 180 311 311 311 F F F 2 040-059 188 188 188 326 326 326 F F F 3 060-079 191 191 191 330 330 330 F F F 4 080-099 191 191 191 330 330 330 F F F 5 100-119 187 187 187 321 321 321 F F F 6 120-139 188 188 188 323 323 323 F F F 7 140-159 191 191 191 330 330 330 F F F 8 160-179 188 188 188 322 322 322 F F F 9 180-199 187 187 187 321 321 321 F F F 10 200-219 187 187 187 321 321 321 F F F 11 220-239 187 187 187 321 321 321 F F F 12 240-259 190 190 190 328 328 328 F F F 13 260-279 191 191 191 330 330 330 F F F 14 280-299 188 188 188 323 323 323 F F F 15 300-319 182 182 182 315 315 315 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 191 191 191 330 330 330 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 191 191 191 330 330 330 F F F 20 400-419 191 191 191 330 330 330 F F F 21 420-439 191 191 191 330 330 330 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 188 188 188 326 326 326 F F F 24 480-499 191 191 191 330 330 330 F F F 25 500-519 190 190 190 329 329 329 F F F 26 520-539 184 184 184 318 318 318 F F F 27 540-559 184 184 184 318 318 318 F F F 28 560-579 187 187 187 321 321 321 F F F 29 580-599 187 187 187 321 321 321 F F F Max gen: 191, Max flush: 330 Begin loop 96 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.098095416Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.143768277Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.144758271Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.144798267Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.144813205Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.144827803Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955} {"msg":"Crucible 639fc108-7f85-432b-bcc5-201e47e62de9 has session id: 6b69565d-1bfd-4862-ae2d-1022eb70dc9f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.14527987Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"Upstairs opts: Upstairs UUID: 639fc108-7f85-432b-bcc5-201e47e62de9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.145328695Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"Crucible stats registered with UUID: 639fc108-7f85-432b-bcc5-201e47e62de9","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.145351742Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176231419Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176281553Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955} {"msg":"639fc108-7f85-432b-bcc5-201e47e62de9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176298671Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176314709Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"0","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176348505Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"0","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176376752Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"1","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176411418Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"1","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.1764747Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"2","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176517295Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"2","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176551821Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176589137Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176623983Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40548)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176658899Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34071)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176691725Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53591)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.176770556Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.177515219Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"0","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.177546885Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"1","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.177632245Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"2","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.178256812Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"2","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.178286639Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"0","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.178304837Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"1","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.178844114Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.178870521Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"[0]R flush_numbers[0..12]: [330, 311, 326, 330, 330, 321, 323, 330, 322, 321, 321, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.178976898Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"[0]R generation[0..12]: [191, 180, 188, 191, 191, 187, 188, 191, 188, 187, 187, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.178997216Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179011934Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"[1]R flush_numbers[0..12]: [330, 311, 326, 330, 330, 321, 323, 330, 322, 321, 321, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179026003Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"[1]R generation[0..12]: [191, 180, 188, 191, 191, 187, 188, 191, 188, 187, 187, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179039831Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179053799Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"[2]R flush_numbers[0..12]: [330, 311, 326, 330, 330, 321, 323, 330, 322, 321, 321, 321]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179067788Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"[2]R generation[0..12]: [191, 180, 188, 191, 191, 187, 188, 191, 188, 187, 187, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179081516Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179095345Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179112723Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179126791Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179140019Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179153118Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"":"downstairs","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179166266Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"Wait for a query_work_queue command to finish before sending IO } {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179185194Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"0","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179198542Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"1","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179211691Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"client":"2","":"downstairs","session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.17922477Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179239378Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"639fc108-7f85-432b-bcc5-201e47e62de9 is now active with session: 6b69565d-1bfd-4862-ae2d-1022eb70dc9f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179263875Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179289262Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955,"session_id":"6b69565d-1bfd-4862-ae2d-1022eb70dc9f"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:44.179315359Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 428 len 28672 data: 16 16 16 13 16 14 13 02/30 Read block 518 len 8192 03/30 Read block 140 len 32768 04/30 Write block 162 len 16384 data: 24 20 19 17 05/30 Read block 303 len 40960 06/30 Read block 405 len 36864 07/30 Write block 525 len 36864 data: 11 12 12 13 11 12 15 17 17 08/30 Write block 92 len 40960 data: 19 23 20 19 17 15 14 11 10 10 09/30 Write block 237 len 8192 data: 11 14 10/30 Flush 11/30 Write block 400 len 32768 data: 18 19 21 21 19 18 19 15 12/30 Write block 490 len 28672 data: 13 11 12 13 16 14 15 13/30 Read block 299 len 24576 14/30 Read block 55 len 24576 15/30 Read block 494 len 4096 16/30 Read block 565 len 24576 17/30 Flush 18/30 Flush 19/30 Write block 552 len 24576 data: 12 12 8 11 10 11 20/30 Read block 79 len 36864 21/30 Read block 263 len 4096 22/30 Read block 338 len 8192 23/30 Write block 71 len 16384 data: 14 12 13 15 24/30 Read block 44 len 40960 25/30 Flush 26/30 Write block 292 len 40960 data: 20 19 20 16 13 16 16 16 15 15 27/30 Read block 336 len 40960 28/30 Read block 98 len 16384 29/30 Write block 29 len 36864 data: 11 9 9 6 8 8 7 10 10 30/30 Read block 428 len 40960 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 5 4 3 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:44.201633585Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1955} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 330 330 330 F F F 1 020-039 180 180 180 311 311 311 T T T 2 040-059 188 188 188 326 326 326 F F F 3 060-079 192 192 192 334 334 334 F F F 4 080-099 192 192 192 331 331 331 F F F 5 100-119 192 192 192 331 331 331 F F F 6 120-139 188 188 188 323 323 323 F F F 7 140-159 191 191 191 330 330 330 F F F 8 160-179 192 192 192 331 331 331 F F F 9 180-199 187 187 187 321 321 321 F F F 10 200-219 187 187 187 321 321 321 F F F 11 220-239 192 192 192 331 331 331 F F F 12 240-259 190 190 190 328 328 328 F F F 13 260-279 191 191 191 330 330 330 F F F 14 280-299 188 188 188 323 323 323 T T T 15 300-319 182 182 182 315 315 315 T T T 16 320-339 188 188 188 323 323 323 F F F 17 340-359 191 191 191 330 330 330 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 191 191 191 330 330 330 F F F 20 400-419 192 192 192 332 332 332 F F F 21 420-439 192 192 192 331 331 331 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 188 188 188 326 326 326 F F F 24 480-499 192 192 192 332 332 332 F F F 25 500-519 190 190 190 329 329 329 F F F 26 520-539 192 192 192 331 331 331 F F F 27 540-559 192 192 192 334 334 334 F F F 28 560-579 187 187 187 321 321 321 F F F 29 580-599 187 187 187 321 321 321 F F F Max gen: 192, Max flush: 334 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 330 330 330 F F F 1 020-039 193 193 193 335 335 335 F F F 2 040-059 188 188 188 326 326 326 F F F 3 060-079 192 192 192 334 334 334 F F F 4 080-099 192 192 192 331 331 331 F F F 5 100-119 192 192 192 331 331 331 F F F 6 120-139 188 188 188 323 323 323 F F F 7 140-159 191 191 191 330 330 330 F F F 8 160-179 192 192 192 331 331 331 F F F 9 180-199 187 187 187 321 321 321 F F F 10 200-219 187 187 187 321 321 321 F F F 11 220-239 192 192 192 331 331 331 F F F 12 240-259 190 190 190 328 328 328 F F F 13 260-279 191 191 191 330 330 330 F F F 14 280-299 193 193 193 335 335 335 F F F 15 300-319 193 193 193 335 335 335 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 191 191 191 330 330 330 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 191 191 191 330 330 330 F F F 20 400-419 192 192 192 332 332 332 F F F 21 420-439 192 192 192 331 331 331 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 188 188 188 326 326 326 F F F 24 480-499 192 192 192 332 332 332 F F F 25 500-519 190 190 190 329 329 329 F F F 26 520-539 192 192 192 331 331 331 F F F 27 540-559 192 192 192 334 334 334 F F F 28 560-579 187 187 187 321 321 321 F F F 29 580-599 187 187 187 321 321 321 F F F Max gen: 193, Max flush: 335 Begin loop 97 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.469707038Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.514310324Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.515142407Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.515190401Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.515208999Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.515227967Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963} {"msg":"Crucible cb4a77ef-d187-4286-a7e8-81fb3124700c has session id: f842ce83-b515-4041-b37d-c2e086a6f4ce","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.515673525Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Upstairs opts: Upstairs UUID: cb4a77ef-d187-4286-a7e8-81fb3124700c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.51571597Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Crucible stats registered with UUID: cb4a77ef-d187-4286-a7e8-81fb3124700c","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.515737617Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546275454Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546318809Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963} {"msg":"cb4a77ef-d187-4286-a7e8-81fb3124700c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546342866Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546359524Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"0","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.54638977Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"0","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546407539Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"1","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546425586Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"1","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546442035Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"2","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546459902Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"2","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546472901Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546487879Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546501637Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:59392)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546640711Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:61850)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546749469Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:63293)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.546799793Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.547614738Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"1","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.547646154Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"2","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.547663272Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"0","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.54836654Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"0","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.548408535Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"1","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.548437492Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"2","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.548987987Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549012655Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"[0]R flush_numbers[0..12]: [330, 335, 326, 334, 331, 331, 323, 330, 331, 321, 321, 331]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549111863Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"[0]R generation[0..12]: [191, 193, 188, 192, 192, 192, 188, 191, 192, 187, 187, 192]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.54913283Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549147859Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"[1]R flush_numbers[0..12]: [330, 335, 326, 334, 331, 331, 323, 330, 331, 321, 321, 331]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549161797Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"[1]R generation[0..12]: [191, 193, 188, 192, 192, 192, 188, 191, 192, 187, 187, 192]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549175476Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549189544Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"[2]R flush_numbers[0..12]: [330, 335, 326, 334, 331, 331, 323, 330, 331, 321, 321, 331]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549206952Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"[2]R generation[0..12]: [191, 193, 188, 192, 192, 192, 188, 191, 192, 187, 187, 192]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.54922116Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549235369Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549248487Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549261715Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549274624Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549287492Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"":"downstairs","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"No downstairs reconciliation required"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.54930721Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549319569Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"0","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549332927Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"1","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549346126Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"client":"2","":"downstairs","session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549359184Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549371443Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"cb4a77ef-d187-4286-a7e8-81fb3124700c is now active with session: f842ce83-b515-4041-b37d-c2e086a6f4ce","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549384211Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.54939631Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963,"session_id":"f842ce83-b515-4041-b37d-c2e086a6f4ce"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:48.549410818Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 301 len 12288 data: 16 18 16 02/30 Read block 144 len 16384 03/30 Write block 490 len 12288 data: 14 12 13 04/30 Read block 14 len 36864 05/30 Flush 06/30 Write block 490 len 28672 data: 15 13 14 14 17 15 16 07/30 Write block 35 len 8192 data: 8 11 08/30 Read block 557 len 8192 09/30 Read block 444 len 40960 10/30 Write block 188 len 16384 data: 13 16 15 17 11/30 Write block 461 len 24576 data: 21 22 20 16 11 12 12/30 Write block 353 len 20480 data: 9 14 12 11 12 13/30 Read block 145 len 36864 14/30 Read block 68 len 20480 15/30 Write block 340 len 8192 data: 14 16 16/30 Read block 102 len 20480 17/30 Write block 195 len 12288 data: 14 13 13 18/30 Read block 121 len 16384 19/30 Flush 20/30 Write block 171 len 20480 data: 12 11 9 10 11 21/30 Write block 170 len 40960 data: 16 13 12 10 11 12 13 15 16 16 22/30 Write block 317 len 20480 data: 9 10 7 12 11 23/30 Flush 24/30 Write block 268 len 24576 data: 12 11 10 9 8 6 25/30 Read block 331 len 16384 26/30 Read block 365 len 8192 27/30 Write block 576 len 24576 data: 15 13 12 10 12 12 28/30 Write block 164 len 36864 data: 20 18 15 18 19 15 17 14 13 29/30 Write block 203 len 32768 data: 16 18 15 17 18 15 18 15 30/30 Read block 420 len 16384 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:48.568554394Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1963} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 330 330 330 F F F 1 020-039 194 194 194 337 337 337 F F F 2 040-059 188 188 188 326 326 326 F F F 3 060-079 192 192 192 334 334 334 F F F 4 080-099 192 192 192 331 331 331 F F F 5 100-119 192 192 192 331 331 331 F F F 6 120-139 188 188 188 323 323 323 F F F 7 140-159 191 191 191 330 330 330 F F F 8 160-179 194 194 194 338 338 338 T T T 9 180-199 194 194 194 337 337 337 F F F 10 200-219 187 187 187 321 321 321 T T T 11 220-239 192 192 192 331 331 331 F F F 12 240-259 190 190 190 328 328 328 F F F 13 260-279 191 191 191 330 330 330 T T T 14 280-299 193 193 193 335 335 335 F F F 15 300-319 194 194 194 338 338 338 F F F 16 320-339 194 194 194 338 338 338 F F F 17 340-359 194 194 194 337 337 337 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 191 191 191 330 330 330 F F F 20 400-419 192 192 192 332 332 332 F F F 21 420-439 192 192 192 331 331 331 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 194 194 194 337 337 337 F F F 24 480-499 194 194 194 337 337 337 F F F 25 500-519 190 190 190 329 329 329 F F F 26 520-539 192 192 192 331 331 331 F F F 27 540-559 192 192 192 334 334 334 F F F 28 560-579 187 187 187 321 321 321 T T T 29 580-599 187 187 187 321 321 321 T T T Max gen: 194, Max flush: 338 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 330 330 330 F F F 1 020-039 194 194 194 337 337 337 F F F 2 040-059 188 188 188 326 326 326 F F F 3 060-079 192 192 192 334 334 334 F F F 4 080-099 192 192 192 331 331 331 F F F 5 100-119 192 192 192 331 331 331 F F F 6 120-139 188 188 188 323 323 323 F F F 7 140-159 191 191 191 330 330 330 F F F 8 160-179 195 195 195 339 339 339 F F F 9 180-199 194 194 194 337 337 337 F F F 10 200-219 195 195 195 339 339 339 F F F 11 220-239 192 192 192 331 331 331 F F F 12 240-259 190 190 190 328 328 328 F F F 13 260-279 195 195 195 339 339 339 F F F 14 280-299 193 193 193 335 335 335 F F F 15 300-319 194 194 194 338 338 338 F F F 16 320-339 194 194 194 338 338 338 F F F 17 340-359 194 194 194 337 337 337 F F F 18 360-379 191 191 191 330 330 330 F F F 19 380-399 191 191 191 330 330 330 F F F 20 400-419 192 192 192 332 332 332 F F F 21 420-439 192 192 192 331 331 331 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 194 194 194 337 337 337 F F F 24 480-499 194 194 194 337 337 337 F F F 25 500-519 190 190 190 329 329 329 F F F 26 520-539 192 192 192 331 331 331 F F F 27 540-559 192 192 192 334 334 334 F F F 28 560-579 195 195 195 339 339 339 F F F 29 580-599 195 195 195 339 339 339 F F F Max gen: 195, Max flush: 339 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.846120909Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:52.891014081Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.891598613Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.923579181Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.924471537Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.924513602Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.92453367Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.924549897Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971} {"msg":"Crucible 5670ded3-42b8-4828-88c9-57a41b55a462 has session id: 2dea85fd-80e1-406a-801f-515a6f443b77","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.925079506Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Upstairs opts: Upstairs UUID: 5670ded3-42b8-4828-88c9-57a41b55a462, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.92512745Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Crucible stats registered with UUID: 5670ded3-42b8-4828-88c9-57a41b55a462","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.925145268Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956036694Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956096236Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971} {"msg":"5670ded3-42b8-4828-88c9-57a41b55a462 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956113054Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956129133Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"0","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956157719Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"0","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956175267Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"1","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956196125Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"1","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956226911Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"2","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956260437Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"2","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956299303Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956334909Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956368755Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"ds_connection connected from Ok(127.0.0.1:60753)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956404851Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"ds_connection connected from Ok(127.0.0.1:49701)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956440056Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"ds_connection connected from Ok(127.0.0.1:54646)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.956476502Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.957286268Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"1","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.957324563Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"2","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.957664193Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"0","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.958119431Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"1","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.958162515Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"2","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.958453272Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"0","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.95880436Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.958838427Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"[0]R flush_numbers[0..12]: [330, 337, 326, 334, 331, 331, 323, 330, 339, 337, 339, 331]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959028694Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"[0]R generation[0..12]: [191, 194, 188, 192, 192, 192, 188, 191, 195, 194, 195, 192]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959052262Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.95906753Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"[1]R flush_numbers[0..12]: [330, 337, 326, 334, 331, 331, 323, 330, 339, 337, 339, 331]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959081758Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"[1]R generation[0..12]: [191, 194, 188, 192, 192, 192, 188, 191, 195, 194, 195, 192]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959095626Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959109725Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"[2]R flush_numbers[0..12]: [330, 337, 326, 334, 331, 331, 323, 330, 339, 337, 339, 331]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959123483Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"[2]R generation[0..12]: [191, 194, 188, 192, 192, 192, 188, 191, 195, 194, 195, 192]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959137312Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.95915259Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959166088Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959179577Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Next flush: 340","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959192545Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959205514Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"":"downstairs","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2025-01-13T15:58:52.959225851Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.95924087Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"0","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959254618Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"1","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959268157Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"client":"2","":"downstairs","session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959281725Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959294073Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"5670ded3-42b8-4828-88c9-57a41b55a462 is now active with session: 2dea85fd-80e1-406a-801f-515a6f443b77","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959307012Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959319051Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971,"session_id":"2dea85fd-80e1-406a-801f-515a6f443b77"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:52.959331229Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 501 len 4096 data: 20 02/30 Read block 330 len 20480 03/30 Write block 172 len 28672 data: 14 11 12 13 14 16 17 04/30 Write block 510 len 24576 data: 15 15 15 16 18 16 05/30 Write block 556 len 32768 data: 11 12 12 13 12 13 15 17 06/30 Write block 376 len 4096 data: 12 07/30 Read block 53 len 32768 08/30 Read block 256 len 28672 09/30 Write block 70 len 16384 data: 16 15 13 14 10/30 Read block 195 len 32768 11/30 Flush 12/30 Write block 98 len 24576 data: 15 12 11 11 11 13 13/30 Write block 75 len 16384 data: 11 7 6 8 14/30 Read block 486 len 4096 15/30 Write block 537 len 8192 data: 19 19 16/30 Read block 282 len 28672 17/30 Write block 139 len 36864 data: 12 17 15 13 14 17 14 12 13 18/30 Write block 503 len 28672 data: 20 18 16 15 14 16 17 19/30 Write block 299 len 40960 data: 17 16 17 19 17 12 11 10 8 7 20/30 Read block 493 len 28672 21/30 Read block 356 len 28672 22/30 Read block 97 len 32768 23/30 Write block 416 len 40960 data: 19 18 16 19 21 21 19 18 17 16 24/30 Read block 502 len 20480 25/30 Write block 396 len 4096 data: 11 26/30 Write block 532 len 40960 data: 18 18 17 15 16 20 20 19 18 18 27/30 Write block 485 len 12288 data: 13 14 15 28/30 Read block 477 len 32768 29/30 Write block 180 len 36864 data: 15 15 14 15 15 13 11 9 14 30/30 Write block 217 len 40960 data: 17 20 16 15 15 15 14 15 12 12 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 6 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 9 Done Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:52.979758895Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1971} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 330 330 330 F F F 1 020-039 194 194 194 337 337 337 F F F 2 040-059 188 188 188 326 326 326 F F F 3 060-079 196 196 196 340 340 340 T T T 4 080-099 192 192 192 331 331 331 T T T 5 100-119 192 192 192 331 331 331 T T T 6 120-139 188 188 188 323 323 323 T T T 7 140-159 191 191 191 330 330 330 T T T 8 160-179 196 196 196 340 340 340 F F F 9 180-199 194 194 194 337 337 337 T T T 10 200-219 195 195 195 339 339 339 T T T 11 220-239 192 192 192 331 331 331 T T T 12 240-259 190 190 190 328 328 328 F F F 13 260-279 195 195 195 339 339 339 F F F 14 280-299 193 193 193 335 335 335 T T T 15 300-319 194 194 194 338 338 338 T T T 16 320-339 194 194 194 338 338 338 F F F 17 340-359 194 194 194 337 337 337 F F F 18 360-379 196 196 196 340 340 340 F F F 19 380-399 191 191 191 330 330 330 T T T 20 400-419 192 192 192 332 332 332 T T T 21 420-439 192 192 192 331 331 331 T T T 22 440-459 188 188 188 326 326 326 F F F 23 460-479 194 194 194 337 337 337 F F F 24 480-499 194 194 194 337 337 337 T T T 25 500-519 196 196 196 340 340 340 T T T 26 520-539 192 192 192 331 331 331 T T T 27 540-559 196 196 196 340 340 340 T T T 28 560-579 196 196 196 340 340 340 F F F 29 580-599 195 195 195 339 339 339 F F F Max gen: 196, Max flush: 340 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 330 330 330 F F F 1 020-039 194 194 194 337 337 337 F F F 2 040-059 188 188 188 326 326 326 F F F 3 060-079 197 197 197 341 341 341 F F F 4 080-099 197 197 197 341 341 341 F F F 5 100-119 197 197 197 341 341 341 F F F 6 120-139 197 197 197 341 341 341 F F F 7 140-159 197 197 197 341 341 341 F F F 8 160-179 196 196 196 340 340 340 F F F 9 180-199 197 197 197 341 341 341 F F F 10 200-219 197 197 197 341 341 341 F F F 11 220-239 197 197 197 341 341 341 F F F 12 240-259 190 190 190 328 328 328 F F F 13 260-279 195 195 195 339 339 339 F F F 14 280-299 197 197 197 341 341 341 F F F 15 300-319 197 197 197 341 341 341 F F F 16 320-339 194 194 194 338 338 338 F F F 17 340-359 194 194 194 337 337 337 F F F 18 360-379 196 196 196 340 340 340 F F F 19 380-399 197 197 197 341 341 341 F F F 20 400-419 197 197 197 341 341 341 F F F 21 420-439 197 197 197 341 341 341 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 194 194 194 337 337 337 F F F 24 480-499 197 197 197 341 341 341 F F F 25 500-519 197 197 197 341 341 341 F F F 26 520-539 197 197 197 341 341 341 F F F 27 540-559 197 197 197 341 341 341 F F F 28 560-579 196 196 196 340 340 340 F F F 29 580-599 195 195 195 339 339 339 F F F Max gen: 197, Max flush: 341 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.374863327Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.419326649Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.420134215Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"12c38e1c2ca92eb8e00f82331ef94fdbf7d3f8af\",\n git_commit_timestamp: \"2025-01-13T15:35:09.000000000Z\",\n git_branch: \"mkeeter/lean-mean-state-machine\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.42017666Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.420191208Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.420206196Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979} {"msg":"Crucible 79a70cf0-e671-46eb-8111-9945b00c5265 has session id: ef49a614-671d-49c5-909a-7d4db0ed5b7f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.420685991Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Upstairs opts: Upstairs UUID: 79a70cf0-e671-46eb-8111-9945b00c5265, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.420734925Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Crucible stats registered with UUID: 79a70cf0-e671-46eb-8111-9945b00c5265","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.420761152Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451414595Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451476688Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979} {"msg":"79a70cf0-e671-46eb-8111-9945b00c5265 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451493996Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451511044Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"0","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.4515394Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"0","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451556678Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"1","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451574576Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"1","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451591154Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"2","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451608822Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"2","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.45162533Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451652347Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451681964Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38190)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451785172Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47785)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451833876Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44776)","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.451893899Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.452660929Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"0","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.452695645Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"2","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.452712194Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"1","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.453369237Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"0","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.45351539Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"1","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.453558405Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"2","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.453925762Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454122579Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"[0]R flush_numbers[0..12]: [330, 337, 326, 341, 341, 341, 341, 341, 340, 341, 341, 341]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454247294Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"[0]R generation[0..12]: [191, 194, 188, 197, 197, 197, 197, 197, 196, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454269242Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.45428507Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"[1]R flush_numbers[0..12]: [330, 337, 326, 341, 341, 341, 341, 341, 340, 341, 341, 341]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454299608Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"[1]R generation[0..12]: [191, 194, 188, 197, 197, 197, 197, 197, 196, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454313896Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454328055Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"[2]R flush_numbers[0..12]: [330, 337, 326, 341, 341, 341, 341, 341, 340, 341, 341, 341]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454341883Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"[2]R generation[0..12]: [191, 194, 188, 197, 197, 197, 197, 197, 196, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454360311Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454374749Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454387998Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454401496Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Next flush: 342","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-13T15:58:57.454414645Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454433183Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"":"downstairs","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454446781Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.45445888Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"0","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454472408Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"1","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454485596Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"client":"2","":"downstairs","session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454498935Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454511353Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"79a70cf0-e671-46eb-8111-9945b00c5265 is now active with session: ef49a614-671d-49c5-909a-7d4db0ed5b7f","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454524232Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454536281Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979,"session_id":"ef49a614-671d-49c5-909a-7d4db0ed5b7f"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-01-13T15:58:57.454549579Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 195 len 4096 data: 15 02/30 Flush 03/30 Write block 230 len 24576 data: 19 18 18 14 14 13 04/30 Read block 329 len 32768 05/30 Write block 253 len 20480 data: 15 17 19 18 19 06/30 Write block 586 len 4096 data: 11 07/30 Read block 234 len 12288 08/30 Read block 577 len 36864 09/30 Write block 55 len 12288 data: 17 13 14 10/30 Write block 306 len 20480 data: 11 9 8 6 7 11/30 Read block 346 len 4096 12/30 Read block 127 len 24576 13/30 Write block 109 len 8192 data: 15 12 14/30 Write block 410 len 12288 data: 20 21 22 15/30 Write block 153 len 40960 data: 17 15 17 16 15 17 18 20 21 25 16/30 Read block 320 len 28672 17/30 Read block 410 len 20480 18/30 Read block 411 len 20480 19/30 Read block 500 len 8192 20/30 Write block 21 len 20480 data: 19 19 21 19 18 21/30 Write block 4 len 20480 data: 10 9 12 16 14 22/30 Read block 324 len 36864 23/30 Read block 515 len 28672 24/30 Read block 420 len 12288 25/30 Read block 294 len 8192 26/30 Read block 510 len 20480 27/30 Read block 301 len 24576 28/30 Read block 476 len 12288 29/30 Read block 395 len 16384 30/30 Write block 321 len 16384 data: 12 11 13 11 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 6 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-13T15:58:57.473641481Z","hostname":"w-01JHG5QE2XHNFWJRVXK8GG2HJ4","pid":1979} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 330 330 330 T T T 1 020-039 194 194 194 337 337 337 T T T 2 040-059 188 188 188 326 326 326 T T T 3 060-079 197 197 197 341 341 341 F F F 4 080-099 197 197 197 341 341 341 F F F 5 100-119 197 197 197 341 341 341 T T T 6 120-139 197 197 197 341 341 341 F F F 7 140-159 197 197 197 341 341 341 T T T 8 160-179 196 196 196 340 340 340 T T T 9 180-199 198 198 198 342 342 342 F F F 10 200-219 197 197 197 341 341 341 F F F 11 220-239 197 197 197 341 341 341 T T T 12 240-259 190 190 190 328 328 328 T T T 13 260-279 195 195 195 339 339 339 F F F 14 280-299 197 197 197 341 341 341 F F F 15 300-319 197 197 197 341 341 341 T T T 16 320-339 194 194 194 338 338 338 T T T 17 340-359 194 194 194 337 337 337 F F F 18 360-379 196 196 196 340 340 340 F F F 19 380-399 197 197 197 341 341 341 F F F 20 400-419 197 197 197 341 341 341 T T T 21 420-439 197 197 197 341 341 341 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 194 194 194 337 337 337 F F F 24 480-499 197 197 197 341 341 341 F F F 25 500-519 197 197 197 341 341 341 F F F 26 520-539 197 197 197 341 341 341 F F F 27 540-559 197 197 197 341 341 341 F F F 28 560-579 196 196 196 340 340 340 F F F 29 580-599 195 195 195 339 339 339 T T T Max gen: 198, Max flush: 342 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 199 199 199 343 343 343 F F F 1 020-039 199 199 199 343 343 343 F F F 2 040-059 199 199 199 343 343 343 F F F 3 060-079 197 197 197 341 341 341 F F F 4 080-099 197 197 197 341 341 341 F F F 5 100-119 199 199 199 343 343 343 F F F 6 120-139 197 197 197 341 341 341 F F F 7 140-159 199 199 199 343 343 343 F F F 8 160-179 199 199 199 343 343 343 F F F 9 180-199 198 198 198 342 342 342 F F F 10 200-219 197 197 197 341 341 341 F F F 11 220-239 199 199 199 343 343 343 F F F 12 240-259 199 199 199 343 343 343 F F F 13 260-279 195 195 195 339 339 339 F F F 14 280-299 197 197 197 341 341 341 F F F 15 300-319 199 199 199 343 343 343 F F F 16 320-339 199 199 199 343 343 343 F F F 17 340-359 194 194 194 337 337 337 F F F 18 360-379 196 196 196 340 340 340 F F F 19 380-399 197 197 197 341 341 341 F F F 20 400-419 199 199 199 343 343 343 F F F 21 420-439 197 197 197 341 341 341 F F F 22 440-459 188 188 188 326 326 326 F F F 23 460-479 194 194 194 337 337 337 F F F 24 480-499 197 197 197 341 341 341 F F F 25 500-519 197 197 197 341 341 341 F F F 26 520-539 197 197 197 341 341 341 F F F 27 540-559 197 197 197 341 341 341 F F F 28 560-579 196 196 196 340 340 340 F F F 29 580-599 199 199 199 343 343 343 F F F Max gen: 199, Max flush: 343 7:21 Test duration Test completed real 7:20.914325564 user 56.111782309 sys 19.567320708 trap 0.079949773 tflt 0.037076011 dflt 0.077847099 kflt 0.006004051 lock 11:44:24.204368089 slp 35:36.987023306 lat 5.590174556 stop 0.064862427