Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.31446245 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.019169172 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.020274876 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1189 1190 1191 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:01.928315643Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:01.99643145Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:01.997696877Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:01.997830644Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:01.997850132Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:01.997865141Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193} {"msg":"Crucible 610b9d30-2421-4298-af26-bae6443de854 has session id: 4908e820-4833-441d-8b13-1d8ca15550ff","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:01.998887881Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Upstairs opts: Upstairs UUID: 610b9d30-2421-4298-af26-bae6443de854, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:01.99900025Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Crucible stats registered with UUID: 610b9d30-2421-4298-af26-bae6443de854","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:01.999020209Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:01.999096701Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000000703Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000119371Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193} {"msg":"610b9d30-2421-4298-af26-bae6443de854 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000142189Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000156638Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"0","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000196844Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"0","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000218182Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"1","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000244129Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"1","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000268787Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"2","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000296374Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"2","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000320942Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000349019Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000378386Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:46442)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000418602Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:49932)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000448579Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:59167)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.000622982Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.002206288Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"2","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.002246674Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"0","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.002282841Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"1","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.003147676Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"1","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.003176394Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"0","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.003192302Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"2","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.003981665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004009593Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004147149Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004168057Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004182676Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004196285Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004209503Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004223142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004236321Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004249559Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004267978Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004281366Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004294435Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004307074Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004319982Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"":"downstairs","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004332881Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.00434475Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"0","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004358169Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"1","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004371267Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"client":"2","":"downstairs","session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:09:02.004384286Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004398205Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"610b9d30-2421-4298-af26-bae6443de854 is now active with session: 4908e820-4833-441d-8b13-1d8ca15550ff","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004415553Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.004431362Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193,"session_id":"4908e820-4833-441d-8b13-1d8ca15550ff"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:02.00444384Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:06.101294198Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1193} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.131403396Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:06.18024809Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.180621533Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.213361255Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.214093234Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.214128801Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.214143429Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.214158168Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195} {"msg":"Crucible 4ff0d7f8-c6a0-45db-bf51-bbed4f6e3f1d has session id: 4f33f2b2-0554-4db0-9a2d-0d649d22501e","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.214761009Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"Upstairs opts: Upstairs UUID: 4ff0d7f8-c6a0-45db-bf51-bbed4f6e3f1d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.214795166Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"Crucible stats registered with UUID: 4ff0d7f8-c6a0-45db-bf51-bbed4f6e3f1d","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.214813244Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:06.214826343Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215276289Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215300057Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195} {"msg":"4ff0d7f8-c6a0-45db-bf51-bbed4f6e3f1d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215360691Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215378359Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"0","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215400047Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"0","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215413746Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"1","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215427654Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"1","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215440523Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"2","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215454352Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"2","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.21546695Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215480669Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215498737Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"ds_connection connected from Ok(127.0.0.1:62969)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215542013Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"io task","client":"2","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41117)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.2155713Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"io task","client":"1","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45643)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.215604587Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"io task","client":"0","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.21639112Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"1","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.216432706Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"2","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.216522248Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"0","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.217022329Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"1","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.217122899Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"0","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.217152396Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"2","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.217826571Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.217852528Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.217955148Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.217975466Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.217989915Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218003663Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218017042Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.21803446Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218060808Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218074996Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218088775Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218101504Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:09:06.218114543Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218187765Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218221122Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"":"downstairs","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.21834145Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218382586Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"0","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218408704Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"1","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218423433Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"client":"2","":"downstairs","session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218436781Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.2184492Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"4ff0d7f8-c6a0-45db-bf51-bbed4f6e3f1d is now active with session: 4f33f2b2-0554-4db0-9a2d-0d649d22501e","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218461999Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218478087Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195,"session_id":"4f33f2b2-0554-4db0-9a2d-0d649d22501e"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:06.218490596Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 578 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Write block 175 len 36864 data: 2 2 2 2 2 2 2 2 2 03/30 Read block 166 len 40960 04/30 Write block 400 len 16384 data: 2 2 2 2 05/30 Flush 06/30 Write block 455 len 24576 data: 2 2 2 2 2 2 07/30 Write block 193 len 4096 data: 2 08/30 Write block 134 len 40960 data: 2 2 2 2 2 2 2 2 2 2 09/30 Read block 440 len 32768 10/30 Flush 11/30 Write block 71 len 8192 data: 2 2 12/30 Read block 533 len 12288 13/30 Read block 163 len 4096 14/30 Read block 366 len 28672 15/30 Read block 158 len 4096 16/30 Write block 433 len 12288 data: 2 2 2 17/30 Read block 542 len 16384 18/30 Write block 405 len 8192 data: 2 2 19/30 Read block 554 len 28672 20/30 Flush 21/30 Write block 203 len 32768 data: 2 2 2 2 2 2 2 2 22/30 Write block 532 len 8192 data: 2 2 23/30 Write block 436 len 20480 data: 2 2 2 2 2 24/30 Read block 186 len 40960 25/30 Write block 379 len 24576 data: 2 2 2 2 2 2 26/30 Flush 27/30 Write block 453 len 20480 data: 2 2 3 3 3 28/30 Read block 230 len 12288 29/30 Read block 432 len 16384 30/30 Read block 102 len 16384 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 3 4 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:06.238084558Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1195} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 2 2 3 3 3 F F F 7 140-159 2 2 2 3 3 3 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 2 2 2 3 3 3 F F F 10 200-219 2 2 2 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 5 5 5 F F F 19 380-399 2 2 2 5 5 5 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 2 2 2 5 5 5 F F F 22 440-459 2 2 2 5 5 5 T T T 23 460-479 2 2 2 3 3 3 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 2, Max flush: 5 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 2 2 3 3 3 F F F 7 140-159 2 2 2 3 3 3 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 2 2 2 3 3 3 F F F 10 200-219 2 2 2 5 5 5 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 5 5 5 F F F 19 380-399 2 2 2 5 5 5 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 2 2 2 5 5 5 F F F 22 440-459 3 3 3 6 6 6 F F F 23 460-479 2 2 2 3 3 3 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 3, Max flush: 6 Begin loop 2 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.486551173Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.529380413Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.530295044Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.530344069Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.530361827Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.530379905Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203} {"msg":"Crucible 87f7cc41-b819-45bb-94fd-ccdc72a7816c has session id: be50ae72-5cdb-48d6-b465-7ec46fac7bb3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.530907634Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Upstairs opts: Upstairs UUID: 87f7cc41-b819-45bb-94fd-ccdc72a7816c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.530955099Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Crucible stats registered with UUID: 87f7cc41-b819-45bb-94fd-ccdc72a7816c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.530980177Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:10.530997605Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531427773Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531451891Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203} {"msg":"87f7cc41-b819-45bb-94fd-ccdc72a7816c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531517964Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531536173Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"0","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.53155862Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"0","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531575679Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"1","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531643832Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"1","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531682808Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"2","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531710816Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"2","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531734403Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.53176444Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531791058Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:47043)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531817235Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"io task","client":"2","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:50588)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531842063Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"io task","client":"0","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:36514)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.531892768Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"io task","client":"1","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.532326506Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"1","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.532460463Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"0","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.532892351Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"1","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.533052455Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"0","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.533484933Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.53362013Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.53393021Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"2","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.534400554Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"2","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 4, 1, 1, 3, 3, 2, 3, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.534863959Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 2, 1, 1, 2, 2, 2, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.534887107Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.534909014Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 4, 1, 1, 3, 3, 2, 3, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.534927432Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 2, 1, 1, 2, 2, 2, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.534944691Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.534962099Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 4, 1, 1, 3, 3, 2, 3, 5, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.534978707Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 2, 1, 1, 2, 2, 2, 2, 2, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.534995286Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535012364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535035532Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.53505154Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535067259Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535083197Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"":"downstairs","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535099596Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535114274Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"0","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535130483Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"1","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535146491Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"client":"2","":"downstairs","session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535164909Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535180308Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"87f7cc41-b819-45bb-94fd-ccdc72a7816c is now active with session: be50ae72-5cdb-48d6-b465-7ec46fac7bb3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535196196Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535210865Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203,"session_id":"be50ae72-5cdb-48d6-b465-7ec46fac7bb3"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:10.535225714Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 232 len 12288 data: 2 2 2 02/30 Read block 538 len 4096 03/30 Write block 445 len 4096 data: 2 04/30 Read block 493 len 28672 05/30 Write block 208 len 4096 data: 3 06/30 Read block 116 len 16384 07/30 Read block 510 len 4096 08/30 Write block 510 len 16384 data: 2 2 2 2 09/30 Write block 93 len 8192 data: 2 2 10/30 Flush 11/30 Write block 296 len 4096 data: 2 12/30 Read block 443 len 16384 13/30 Flush 14/30 Write block 353 len 24576 data: 2 2 2 2 2 2 15/30 Write block 157 len 16384 data: 2 2 2 2 16/30 Write block 577 len 36864 data: 2 3 3 3 3 3 3 3 3 17/30 Read block 385 len 4096 18/30 Read block 98 len 8192 19/30 Read block 320 len 24576 20/30 Flush 21/30 Write block 376 len 24576 data: 2 2 2 3 3 3 22/30 Read block 526 len 36864 23/30 Read block 188 len 12288 24/30 Read block 100 len 36864 25/30 Write block 89 len 20480 data: 2 2 2 2 3 26/30 Write block 250 len 32768 data: 2 2 2 2 2 2 2 2 27/30 Write block 228 len 12288 data: 2 2 2 28/30 Read block 425 len 36864 29/30 Write block 338 len 32768 data: 2 2 2 2 2 2 2 2 30/30 Read block 322 len 40960 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 9 10 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:10.552976345Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1203} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 4 4 4 7 7 7 T T T 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 2 2 3 3 3 F F F 7 140-159 4 4 4 9 9 9 F F F 8 160-179 4 4 4 9 9 9 F F F 9 180-199 2 2 2 3 3 3 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 4 4 4 7 7 7 T T T 12 240-259 1 1 1 1 1 1 T T T 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 T T T 17 340-359 4 4 4 9 9 9 T T T 18 360-379 2 2 2 5 5 5 T T T 19 380-399 2 2 2 5 5 5 T T T 20 400-419 2 2 2 4 4 4 F F F 21 420-439 2 2 2 5 5 5 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 2 2 2 3 3 3 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 4 4 4 7 7 7 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 4 4 4 9 9 9 F F F 29 580-599 4 4 4 9 9 9 F F F Max gen: 4, Max flush: 9 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 5 5 5 10 10 10 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 2 2 3 3 3 F F F 7 140-159 4 4 4 9 9 9 F F F 8 160-179 4 4 4 9 9 9 F F F 9 180-199 2 2 2 3 3 3 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 5 5 5 10 10 10 F F F 12 240-259 5 5 5 10 10 10 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 5 5 5 10 10 10 F F F 17 340-359 5 5 5 10 10 10 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 5 5 5 10 10 10 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 2 2 2 5 5 5 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 2 2 2 3 3 3 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 4 4 4 7 7 7 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 4 4 4 9 9 9 F F F 29 580-599 4 4 4 9 9 9 F F F Max gen: 5, Max flush: 10 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.791987289Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.835430429Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.836007853Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.836049299Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.836064207Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.836078576Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211} {"msg":"Crucible 71b5f9aa-0856-4187-9b6e-f3c3ccf73b4b has session id: 510d0978-e6ab-4e7b-b0c3-8588d7d966c0","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.836595456Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Upstairs opts: Upstairs UUID: 71b5f9aa-0856-4187-9b6e-f3c3ccf73b4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.836641001Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Crucible stats registered with UUID: 71b5f9aa-0856-4187-9b6e-f3c3ccf73b4b","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.836667129Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:14.836688477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837145162Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.83717202Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211} {"msg":"71b5f9aa-0856-4187-9b6e-f3c3ccf73b4b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837207766Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837225004Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"0","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837255441Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"0","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837282279Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"1","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837316395Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"1","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837345873Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"2","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.83737551Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"2","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837402707Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"io task","client":"2","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837431364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"io task","client":"0","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837459921Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"io task","client":"1","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:42629)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837486489Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"io task","client":"2","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:59844)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837600808Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"io task","client":"0","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:52573)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.837697058Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"io task","client":"1","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.838176312Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"2","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.838213578Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"0","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.838243625Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"1","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.838799031Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"2","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.838838977Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"1","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.838934158Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"0","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839344398Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839440909Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 4, 10, 1, 3, 9, 9, 3, 7, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839502633Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 2, 5, 1, 2, 4, 4, 2, 4, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839522371Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839537589Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 4, 10, 1, 3, 9, 9, 3, 7, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839551098Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 2, 5, 1, 2, 4, 4, 2, 4, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839570576Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839596133Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 4, 10, 1, 3, 9, 9, 3, 7, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839621521Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 2, 5, 1, 2, 4, 4, 2, 4, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839639989Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839669986Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839695034Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839710282Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839726441Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839753868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"":"downstairs","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839780486Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839797394Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"0","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839810672Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"1","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.83983147Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"client":"2","":"downstairs","session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839855998Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839879446Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"71b5f9aa-0856-4187-9b6e-f3c3ccf73b4b is now active with session: 510d0978-e6ab-4e7b-b0c3-8588d7d966c0","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839896364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:14.839908353Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211,"session_id":"510d0978-e6ab-4e7b-b0c3-8588d7d966c0"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-01-28T17:09:14.839928731Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 107 len 20480 data: 2 2 2 2 2 02/30 Read block 37 len 36864 03/30 Read block 298 len 40960 04/30 Write block 482 len 36864 data: 2 2 2 2 2 2 2 2 2 05/30 Flush 06/30 Write block 507 len 4096 data: 2 07/30 Write block 589 len 4096 data: 2 08/30 Read block 559 len 4096 09/30 Write block 482 len 28672 data: 3 3 3 3 3 3 3 10/30 Write block 577 len 8192 data: 3 4 11/30 Flush 12/30 Write block 585 len 8192 data: 4 2 13/30 Flush 14/30 Write block 108 len 8192 data: 3 3 15/30 Read block 168 len 36864 16/30 Write block 550 len 20480 data: 2 2 2 2 2 17/30 Read block 280 len 24576 18/30 Write block 174 len 16384 data: 2 3 3 3 19/30 Read block 12 len 32768 20/30 Write block 357 len 36864 data: 3 3 2 2 2 2 2 2 2 21/30 Write block 362 len 28672 data: 3 3 3 3 2 2 2 22/30 Write block 262 len 40960 data: 2 2 2 2 2 2 2 2 2 2 23/30 Write block 245 len 40960 data: 2 2 2 2 2 3 3 3 3 3 24/30 Write block 500 len 24576 data: 2 2 2 2 2 2 25/30 Write block 135 len 20480 data: 3 3 3 3 3 26/30 Read block 483 len 8192 27/30 Write block 464 len 24576 data: 2 2 2 2 2 2 28/30 Read block 400 len 16384 29/30 Read block 157 len 28672 30/30 Write block 341 len 32768 data: 3 3 3 3 3 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 2 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:14.857978014Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1211} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 5 5 5 10 10 10 F F F 5 100-119 6 6 6 11 11 11 T T T 6 120-139 2 2 2 3 3 3 T T T 7 140-159 4 4 4 9 9 9 F F F 8 160-179 4 4 4 9 9 9 T T T 9 180-199 2 2 2 3 3 3 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 5 5 5 10 10 10 F F F 12 240-259 5 5 5 10 10 10 T T T 13 260-279 1 1 1 1 1 1 T T T 14 280-299 4 4 4 8 8 8 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 5 5 5 10 10 10 F F F 17 340-359 5 5 5 10 10 10 T T T 18 360-379 5 5 5 10 10 10 T T T 19 380-399 5 5 5 10 10 10 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 2 2 2 5 5 5 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 2 2 2 3 3 3 T T T 24 480-499 6 6 6 12 12 12 F F F 25 500-519 6 6 6 12 12 12 T T T 26 520-539 2 2 2 5 5 5 F F F 27 540-559 1 1 1 1 1 1 T T T 28 560-579 6 6 6 12 12 12 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 6, Max flush: 13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 5 5 5 10 10 10 F F F 5 100-119 7 7 7 14 14 14 F F F 6 120-139 7 7 7 14 14 14 F F F 7 140-159 4 4 4 9 9 9 F F F 8 160-179 7 7 7 14 14 14 F F F 9 180-199 2 2 2 3 3 3 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 5 5 5 10 10 10 F F F 12 240-259 7 7 7 14 14 14 F F F 13 260-279 7 7 7 14 14 14 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 5 5 5 10 10 10 F F F 17 340-359 7 7 7 14 14 14 F F F 18 360-379 7 7 7 14 14 14 F F F 19 380-399 5 5 5 10 10 10 F F F 20 400-419 2 2 2 4 4 4 F F F 21 420-439 2 2 2 5 5 5 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 7 7 7 14 14 14 F F F 24 480-499 6 6 6 12 12 12 F F F 25 500-519 7 7 7 14 14 14 F F F 26 520-539 2 2 2 5 5 5 F F F 27 540-559 7 7 7 14 14 14 F F F 28 560-579 6 6 6 12 12 12 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 7, Max flush: 14 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.118461079Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:19.161983651Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.162395011Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.194278957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.194883178Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.194916704Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.194930483Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.194944622Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219} {"msg":"Crucible ada8c5e5-e66e-4783-8697-13b68c03830b has session id: b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.195593338Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Upstairs opts: Upstairs UUID: ada8c5e5-e66e-4783-8697-13b68c03830b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.195627595Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Crucible stats registered with UUID: ada8c5e5-e66e-4783-8697-13b68c03830b","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.195645793Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:19.195658772Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196063713Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196101269Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219} {"msg":"ada8c5e5-e66e-4783-8697-13b68c03830b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196131166Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196154004Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"0","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.19618984Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"0","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196223127Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"1","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196253294Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"1","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196279172Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"2","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196303919Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"2","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196322618Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196336976Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196350445Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:57273)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196365193Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"io task","client":"0","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:40914)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.19639559Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"io task","client":"2","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:33358)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.196421438Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"io task","client":"1","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.197145207Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"1","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.197171835Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"2","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.197186193Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"0","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.197803583Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"1","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.197831031Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"0","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.197846019Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"2","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198381327Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198406364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 4, 10, 14, 14, 9, 14, 3, 7, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198430002Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 2, 5, 7, 7, 4, 7, 2, 4, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198444361Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198458499Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 4, 10, 14, 14, 9, 14, 3, 7, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198472208Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 2, 5, 7, 7, 4, 7, 2, 4, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198489136Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198503095Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 4, 10, 14, 14, 9, 14, 3, 7, 10]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198516544Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 2, 5, 7, 7, 4, 7, 2, 4, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198529983Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198543671Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.19855621Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198569409Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198582138Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198594676Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"":"downstairs","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198607295Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:09:19.198618884Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"0","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198636072Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"1","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198648881Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"client":"2","":"downstairs","session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.19866172Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198673598Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"ada8c5e5-e66e-4783-8697-13b68c03830b is now active with session: b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198688237Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198700156Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219,"session_id":"b0a5cf4e-08a9-4aa2-a44f-b5c28b8d49f2"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:19.198712185Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 189 len 36864 data: 2 2 2 2 3 2 2 2 2 02/30 Read block 516 len 20480 03/30 Write block 529 len 12288 data: 2 2 2 04/30 Write block 524 len 40960 data: 2 2 2 2 2 3 3 3 3 3 05/30 Read block 591 len 24576 06/30 Write block 470 len 16384 data: 2 2 2 2 07/30 Read block 159 len 32768 08/30 Write block 392 len 8192 data: 2 2 09/30 Flush 10/30 Write block 192 len 4096 data: 3 11/30 Write block 264 len 16384 data: 3 3 3 3 12/30 Write block 171 len 12288 data: 2 2 2 13/30 Write block 251 len 32768 data: 4 4 4 4 3 3 3 2 14/30 Write block 416 len 8192 data: 2 2 15/30 Read block 457 len 4096 16/30 Write block 346 len 32768 data: 3 3 3 2 2 2 2 3 17/30 Write block 300 len 40960 data: 2 2 2 2 2 2 2 2 2 2 18/30 Read block 252 len 24576 19/30 Read block 350 len 8192 20/30 Write block 37 len 16384 data: 2 2 2 2 21/30 Flush 22/30 Flush 23/30 Write block 131 len 40960 data: 2 2 2 3 4 4 4 4 4 3 24/30 Flush 25/30 Write block 340 len 36864 data: 3 4 4 4 4 4 4 4 4 26/30 Write block 391 len 12288 data: 2 3 3 27/30 Write block 7 len 8192 data: 2 2 28/30 Write block 462 len 24576 data: 2 2 3 3 3 3 29/30 Write block 239 len 40960 data: 2 2 2 2 2 2 3 3 3 3 30/30 Write block 363 len 24576 data: 4 4 4 3 3 3 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:19.216583745Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1219} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T T 1 020-039 8 8 8 16 16 16 F F F 2 040-059 8 8 8 16 16 16 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 5 5 5 10 10 10 F F F 5 100-119 7 7 7 14 14 14 F F F 6 120-139 8 8 8 18 18 18 F F F 7 140-159 8 8 8 18 18 18 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 8 8 8 16 16 16 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 5 5 5 10 10 10 T T T 12 240-259 8 8 8 16 16 16 T T T 13 260-279 8 8 8 16 16 16 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 8 8 8 16 16 16 F F F 16 320-339 5 5 5 10 10 10 F F F 17 340-359 8 8 8 16 16 16 T T T 18 360-379 7 7 7 14 14 14 T T T 19 380-399 8 8 8 15 15 15 T T T 20 400-419 8 8 8 16 16 16 F F F 21 420-439 2 2 2 5 5 5 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 8 8 8 15 15 15 T T T 24 480-499 6 6 6 12 12 12 F F F 25 500-519 7 7 7 14 14 14 F F F 26 520-539 8 8 8 15 15 15 F F F 27 540-559 7 7 7 14 14 14 F F F 28 560-579 6 6 6 12 12 12 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 8, Max flush: 18 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 19 19 19 F F F 1 020-039 8 8 8 16 16 16 F F F 2 040-059 8 8 8 16 16 16 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 5 5 5 10 10 10 F F F 5 100-119 7 7 7 14 14 14 F F F 6 120-139 8 8 8 18 18 18 F F F 7 140-159 8 8 8 18 18 18 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 8 8 8 16 16 16 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 9 9 9 19 19 19 F F F 12 240-259 9 9 9 19 19 19 F F F 13 260-279 8 8 8 16 16 16 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 8 8 8 16 16 16 F F F 16 320-339 5 5 5 10 10 10 F F F 17 340-359 9 9 9 19 19 19 F F F 18 360-379 9 9 9 19 19 19 F F F 19 380-399 9 9 9 19 19 19 F F F 20 400-419 8 8 8 16 16 16 F F F 21 420-439 2 2 2 5 5 5 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 9 9 9 19 19 19 F F F 24 480-499 6 6 6 12 12 12 F F F 25 500-519 7 7 7 14 14 14 F F F 26 520-539 8 8 8 15 15 15 F F F 27 540-559 7 7 7 14 14 14 F F F 28 560-579 6 6 6 12 12 12 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 9, Max flush: 19 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.489108396Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.532907541Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.533672077Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.533713823Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.533732091Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.533747319Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227} {"msg":"Crucible 1b4c1690-f1e3-476d-9701-ccce875d65a3 has session id: 9875dbb1-afe7-4205-bb7d-ccca28f71046","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.534307445Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Upstairs opts: Upstairs UUID: 1b4c1690-f1e3-476d-9701-ccce875d65a3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.534344911Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Crucible stats registered with UUID: 1b4c1690-f1e3-476d-9701-ccce875d65a3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.534366689Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:23.534380378Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.534794468Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.534821505Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227} {"msg":"1b4c1690-f1e3-476d-9701-ccce875d65a3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.534881529Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.534900107Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"0","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.534926065Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"0","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.534951472Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"1","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.535047663Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"1","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.535138234Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"2","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.53518225Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"2","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.535218716Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.535248423Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.535274921Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"ds_connection connected from Ok(127.0.0.1:37834)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.535303378Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"io task","client":"0","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"ds_connection connected from Ok(127.0.0.1:64788)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.535332615Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"io task","client":"2","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"ds_connection connected from Ok(127.0.0.1:47418)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.535357753Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"io task","client":"1","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.535786141Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"1","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.535815888Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"0","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.535945116Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"2","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.536433148Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"1","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.536460515Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"0","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.536502511Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"2","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537036299Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537139009Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"[0]R flush_numbers[0..12]: [19, 16, 16, 4, 10, 14, 18, 18, 16, 16, 7, 19]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537201643Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"[0]R generation[0..12]: [9, 8, 8, 2, 5, 7, 8, 8, 8, 8, 4, 9]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537221381Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.53723564Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"[1]R flush_numbers[0..12]: [19, 16, 16, 4, 10, 14, 18, 18, 16, 16, 7, 19]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537249209Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"[1]R generation[0..12]: [9, 8, 8, 2, 5, 7, 8, 8, 8, 8, 4, 9]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537262447Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537276106Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"[2]R flush_numbers[0..12]: [19, 16, 16, 4, 10, 14, 18, 18, 16, 16, 7, 19]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537289465Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"[2]R generation[0..12]: [9, 8, 8, 2, 5, 7, 8, 8, 8, 8, 4, 9]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537302583Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537316112Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537328641Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537341269Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:09:23.537353868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537370807Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"":"downstairs","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537383565Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537397614Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"0","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537410633Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"1","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537423412Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"client":"2","":"downstairs","session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.53743626Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537448339Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"1b4c1690-f1e3-476d-9701-ccce875d65a3 is now active with session: 9875dbb1-afe7-4205-bb7d-ccca28f71046","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537460708Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537472337Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227,"session_id":"9875dbb1-afe7-4205-bb7d-ccca28f71046"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:23.537484226Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 520 len 24576 data: 2 2 2 2 3 3 03/30 Read block 478 len 40960 04/30 Write block 550 len 4096 data: 3 05/30 Read block 429 len 28672 06/30 Write block 265 len 28672 data: 4 4 4 3 3 3 3 07/30 Write block 278 len 24576 data: 2 2 2 2 2 2 08/30 Write block 16 len 16384 data: 2 2 2 2 09/30 Write block 512 len 24576 data: 3 3 2 2 2 2 10/30 Read block 4 len 4096 11/30 Read block 572 len 12288 12/30 Read block 584 len 20480 13/30 Write block 309 len 40960 data: 3 2 2 2 2 2 2 2 2 2 14/30 Flush 15/30 Flush 16/30 Flush 17/30 Write block 177 len 16384 data: 4 3 3 3 18/30 Write block 177 len 36864 data: 5 4 4 4 3 3 3 2 2 19/30 Flush 20/30 Write block 434 len 4096 data: 3 21/30 Read block 96 len 28672 22/30 Write block 452 len 16384 data: 2 3 3 4 23/30 Write block 337 len 4096 data: 2 24/30 Write block 337 len 36864 data: 3 3 3 4 5 5 5 5 5 25/30 Read block 133 len 4096 26/30 Read block 211 len 4096 27/30 Write block 180 len 32768 data: 5 4 4 4 3 3 2 2 28/30 Write block 477 len 36864 data: 2 2 2 2 2 4 4 4 4 29/30 Read block 456 len 40960 30/30 Write block 347 len 32768 data: 5 5 3 3 3 3 4 3 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 1 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:23.556179895Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1227} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 21 21 21 F F F 1 020-039 8 8 8 16 16 16 F F F 2 040-059 8 8 8 16 16 16 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 5 5 5 10 10 10 F F F 5 100-119 7 7 7 14 14 14 F F F 6 120-139 8 8 8 18 18 18 F F F 7 140-159 8 8 8 18 18 18 F F F 8 160-179 10 10 10 24 24 24 F F F 9 180-199 10 10 10 24 24 24 T T T 10 200-219 4 4 4 7 7 7 F F F 11 220-239 9 9 9 19 19 19 F F F 12 240-259 9 9 9 19 19 19 F F F 13 260-279 10 10 10 21 21 21 F F F 14 280-299 10 10 10 21 21 21 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 5 5 5 10 10 10 T T T 17 340-359 9 9 9 19 19 19 T T T 18 360-379 9 9 9 19 19 19 F F F 19 380-399 9 9 9 19 19 19 F F F 20 400-419 8 8 8 16 16 16 F F F 21 420-439 2 2 2 5 5 5 T T T 22 440-459 4 4 4 7 7 7 T T T 23 460-479 9 9 9 19 19 19 T T T 24 480-499 6 6 6 12 12 12 T T T 25 500-519 10 10 10 21 21 21 F F F 26 520-539 10 10 10 21 21 21 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 6 6 6 12 12 12 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 10, Max flush: 24 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 21 21 21 F F F 1 020-039 8 8 8 16 16 16 F F F 2 040-059 8 8 8 16 16 16 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 5 5 5 10 10 10 F F F 5 100-119 7 7 7 14 14 14 F F F 6 120-139 8 8 8 18 18 18 F F F 7 140-159 8 8 8 18 18 18 F F F 8 160-179 10 10 10 24 24 24 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 9 9 9 19 19 19 F F F 12 240-259 9 9 9 19 19 19 F F F 13 260-279 10 10 10 21 21 21 F F F 14 280-299 10 10 10 21 21 21 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 11 11 11 25 25 25 F F F 17 340-359 11 11 11 25 25 25 F F F 18 360-379 9 9 9 19 19 19 F F F 19 380-399 9 9 9 19 19 19 F F F 20 400-419 8 8 8 16 16 16 F F F 21 420-439 11 11 11 25 25 25 F F F 22 440-459 11 11 11 25 25 25 F F F 23 460-479 11 11 11 25 25 25 F F F 24 480-499 11 11 11 25 25 25 F F F 25 500-519 10 10 10 21 21 21 F F F 26 520-539 10 10 10 21 21 21 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 6 6 6 12 12 12 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 11, Max flush: 25 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.797399915Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.840624326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.841239027Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.841281003Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.841297261Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.84131172Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235} {"msg":"Crucible 7448193f-e6c9-4310-a164-ff34de14d987 has session id: f0d1dadd-ef9c-4342-918a-d953d74528a1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.841790403Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Upstairs opts: Upstairs UUID: 7448193f-e6c9-4310-a164-ff34de14d987, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.841838798Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Crucible stats registered with UUID: 7448193f-e6c9-4310-a164-ff34de14d987","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.841863996Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:27.841883774Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842479516Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842503124Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235} {"msg":"7448193f-e6c9-4310-a164-ff34de14d987 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842569697Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842595955Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"0","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842625062Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"0","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.84264125Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"1","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842655929Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"1","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842678177Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"2","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842703924Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"2","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842727062Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.8427465Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842760409Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47460)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842867628Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"io task","client":"2","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:54388)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842907904Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"io task","client":"0","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"ds_connection connected from Ok(127.0.0.1:48832)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.842933672Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"io task","client":"1","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.843579159Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"0","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.843615665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"1","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.844204758Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"1","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.844359943Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"0","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.844711279Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.844912979Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.844988182Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"2","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.84552484Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"2","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"[0]R flush_numbers[0..12]: [21, 16, 16, 4, 10, 14, 18, 18, 24, 25, 7, 19]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846022591Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"[0]R generation[0..12]: [10, 8, 8, 2, 5, 7, 8, 8, 10, 11, 4, 9]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846043579Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846058617Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"[1]R flush_numbers[0..12]: [21, 16, 16, 4, 10, 14, 18, 18, 24, 25, 7, 19]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846072206Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"[1]R generation[0..12]: [10, 8, 8, 2, 5, 7, 8, 8, 10, 11, 4, 9]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846085315Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846098853Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"[2]R flush_numbers[0..12]: [21, 16, 16, 4, 10, 14, 18, 18, 24, 25, 7, 19]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846112212Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"[2]R generation[0..12]: [10, 8, 8, 2, 5, 7, 8, 8, 10, 11, 4, 9]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846125211Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.84613888Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:09:27.846155498Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846170726Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846183385Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846195904Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"":"downstairs","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846208743Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846220352Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"0","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.84623328Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"1","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846246009Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"client":"2","":"downstairs","session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846258718Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-01-28T17:09:27.846270477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"7448193f-e6c9-4310-a164-ff34de14d987 is now active with session: f0d1dadd-ef9c-4342-918a-d953d74528a1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846286465Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846298224Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235,"session_id":"f0d1dadd-ef9c-4342-918a-d953d74528a1"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:27.846310163Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 400 len 16384 data: 3 3 3 3 02/30 Write block 226 len 12288 data: 2 2 3 03/30 Write block 164 len 24576 data: 2 2 2 2 2 2 04/30 Flush 05/30 Write block 168 len 28672 data: 3 3 2 3 3 3 3 06/30 Write block 314 len 8192 data: 3 3 07/30 Read block 217 len 32768 08/30 Write block 267 len 12288 data: 5 4 4 09/30 Read block 375 len 4096 10/30 Flush 11/30 Write block 90 len 8192 data: 3 3 12/30 Read block 403 len 40960 13/30 Read block 22 len 36864 14/30 Read block 248 len 20480 15/30 Write block 225 len 20480 data: 2 3 3 4 3 16/30 Write block 306 len 24576 data: 3 3 3 4 3 3 17/30 Write block 489 len 8192 data: 3 3 18/30 Flush 19/30 Write block 37 len 12288 data: 3 3 3 20/30 Flush 21/30 Write block 530 len 16384 data: 4 4 4 4 22/30 Write block 403 len 28672 data: 4 2 3 3 2 2 2 23/30 Read block 74 len 12288 24/30 Write block 52 len 40960 data: 2 2 2 2 2 2 2 2 2 2 25/30 Write block 529 len 12288 data: 4 5 5 26/30 Flush 27/30 Write block 513 len 24576 data: 4 3 3 3 3 2 28/30 Write block 519 len 24576 data: 2 3 3 3 3 4 29/30 Read block 12 len 32768 30/30 Read block 52 len 24576 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 3 3 4 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:27.864284463Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1235} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 21 21 21 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 12 12 12 30 30 30 F F F 3 060-079 12 12 12 30 30 30 F F F 4 080-099 12 12 12 28 28 28 F F F 5 100-119 7 7 7 14 14 14 F F F 6 120-139 8 8 8 18 18 18 F F F 7 140-159 8 8 8 18 18 18 F F F 8 160-179 12 12 12 27 27 27 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 12 12 12 28 28 28 F F F 12 240-259 9 9 9 19 19 19 F F F 13 260-279 12 12 12 27 27 27 F F F 14 280-299 10 10 10 21 21 21 F F F 15 300-319 12 12 12 28 28 28 F F F 16 320-339 11 11 11 25 25 25 F F F 17 340-359 11 11 11 25 25 25 F F F 18 360-379 9 9 9 19 19 19 F F F 19 380-399 9 9 9 19 19 19 F F F 20 400-419 12 12 12 30 30 30 F F F 21 420-439 11 11 11 25 25 25 F F F 22 440-459 11 11 11 25 25 25 F F F 23 460-479 11 11 11 25 25 25 F F F 24 480-499 12 12 12 28 28 28 F F F 25 500-519 10 10 10 21 21 21 T T T 26 520-539 12 12 12 30 30 30 T T T 27 540-559 10 10 10 21 21 21 F F F 28 560-579 6 6 6 12 12 12 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 12, Max flush: 30 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 21 21 21 F F F 1 020-039 12 12 12 29 29 29 F F F 2 040-059 12 12 12 30 30 30 F F F 3 060-079 12 12 12 30 30 30 F F F 4 080-099 12 12 12 28 28 28 F F F 5 100-119 7 7 7 14 14 14 F F F 6 120-139 8 8 8 18 18 18 F F F 7 140-159 8 8 8 18 18 18 F F F 8 160-179 12 12 12 27 27 27 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 12 12 12 28 28 28 F F F 12 240-259 9 9 9 19 19 19 F F F 13 260-279 12 12 12 27 27 27 F F F 14 280-299 10 10 10 21 21 21 F F F 15 300-319 12 12 12 28 28 28 F F F 16 320-339 11 11 11 25 25 25 F F F 17 340-359 11 11 11 25 25 25 F F F 18 360-379 9 9 9 19 19 19 F F F 19 380-399 9 9 9 19 19 19 F F F 20 400-419 12 12 12 30 30 30 F F F 21 420-439 11 11 11 25 25 25 F F F 22 440-459 11 11 11 25 25 25 F F F 23 460-479 11 11 11 25 25 25 F F F 24 480-499 12 12 12 28 28 28 F F F 25 500-519 13 13 13 31 31 31 F F F 26 520-539 13 13 13 31 31 31 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 6 6 6 12 12 12 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 13, Max flush: 31 Begin loop 7 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.082371656Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:32.124123821Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.124511353Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.155182246Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.155772999Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.155816075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.155834943Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.155853071Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243} {"msg":"Crucible 1171f942-f127-46ce-b787-f2043ae971ef has session id: dc9b063e-ec94-4c37-95b9-b801b51fba8f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.156451083Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"Upstairs opts: Upstairs UUID: 1171f942-f127-46ce-b787-f2043ae971ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.156487909Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"Crucible stats registered with UUID: 1171f942-f127-46ce-b787-f2043ae971ef","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.156507468Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:32.156520736Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.156887Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.156909898Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243} {"msg":"1171f942-f127-46ce-b787-f2043ae971ef active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.156967493Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.156985311Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"0","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.157010439Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"0","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.157035746Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"1","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.157063513Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"1","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.157090811Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"2","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.157119018Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"2","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.157146665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.157174903Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.15720284Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:35809)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.157238936Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"io task","client":"2","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63063)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.157268433Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"io task","client":"1","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42731)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.157291861Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"io task","client":"0","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.157948727Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"1","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.157986903Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"0","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.15801753Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"2","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.158594394Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"1","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.15873736Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"0","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.158764328Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"2","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159130202Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159329323Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"[0]R flush_numbers[0..12]: [21, 29, 30, 30, 28, 14, 18, 18, 27, 25, 7, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159451451Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"[0]R generation[0..12]: [10, 12, 12, 12, 12, 7, 8, 8, 12, 11, 4, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159471569Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159486447Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"[1]R flush_numbers[0..12]: [21, 29, 30, 30, 28, 14, 18, 18, 27, 25, 7, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159500466Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"[1]R generation[0..12]: [10, 12, 12, 12, 12, 7, 8, 8, 12, 11, 4, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159518204Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159532403Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"[2]R flush_numbers[0..12]: [21, 29, 30, 30, 28, 14, 18, 18, 27, 25, 7, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159546081Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"[2]R generation[0..12]: [10, 12, 12, 12, 12, 7, 8, 8, 12, 11, 4, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.15955954Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159573509Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159592187Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159605126Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159617865Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159630493Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"":"downstairs","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159643442Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159655321Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"0","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.15966845Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"1","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159681248Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"client":"2","":"downstairs","session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159694167Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159709486Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"1171f942-f127-46ce-b787-f2043ae971ef is now active with session: dc9b063e-ec94-4c37-95b9-b801b51fba8f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159724964Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159737013Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243,"session_id":"dc9b063e-ec94-4c37-95b9-b801b51fba8f"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:32.159748992Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 155 len 40960 data: 2 2 3 3 3 3 2 2 2 3 02/30 Read block 4 len 20480 03/30 Write block 41 len 16384 data: 2 2 2 2 04/30 Write block 65 len 28672 data: 2 2 2 2 2 2 3 05/30 Read block 225 len 12288 06/30 Write block 364 len 8192 data: 5 5 07/30 Read block 485 len 12288 08/30 Read block 398 len 16384 09/30 Flush 10/30 Write block 21 len 24576 data: 2 2 2 2 2 2 11/30 Read block 562 len 24576 12/30 Write block 79 len 28672 data: 2 2 2 2 2 2 2 13/30 Read block 115 len 28672 14/30 Read block 14 len 20480 15/30 Read block 578 len 32768 16/30 Write block 235 len 32768 data: 2 2 2 2 3 3 3 3 17/30 Read block 90 len 24576 18/30 Read block 588 len 24576 19/30 Write block 168 len 12288 data: 4 4 3 20/30 Flush 21/30 Write block 491 len 20480 data: 2 2 2 2 2 22/30 Read block 500 len 36864 23/30 Flush 24/30 Flush 25/30 Write block 66 len 28672 data: 3 3 3 3 3 4 3 26/30 Write block 171 len 24576 data: 4 4 4 4 4 4 27/30 Write block 478 len 24576 data: 3 3 3 3 5 5 28/30 Read block 251 len 24576 29/30 Write block 172 len 12288 data: 5 5 5 30/30 Write block 4 len 16384 data: 2 2 2 3 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:32.179408767Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1243} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 10 21 21 21 T T T 1 020-039 14 14 14 33 33 33 F F F 2 040-059 14 14 14 32 32 32 F F F 3 060-079 14 14 14 33 33 33 T T T 4 080-099 14 14 14 33 33 33 F F F 5 100-119 7 7 7 14 14 14 F F F 6 120-139 8 8 8 18 18 18 F F F 7 140-159 14 14 14 32 32 32 F F F 8 160-179 14 14 14 33 33 33 T T T 9 180-199 11 11 11 25 25 25 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 14 14 14 33 33 33 F F F 12 240-259 14 14 14 33 33 33 F F F 13 260-279 12 12 12 27 27 27 F F F 14 280-299 10 10 10 21 21 21 F F F 15 300-319 12 12 12 28 28 28 F F F 16 320-339 11 11 11 25 25 25 F F F 17 340-359 11 11 11 25 25 25 F F F 18 360-379 14 14 14 32 32 32 F F F 19 380-399 9 9 9 19 19 19 F F F 20 400-419 12 12 12 30 30 30 F F F 21 420-439 11 11 11 25 25 25 F F F 22 440-459 11 11 11 25 25 25 F F F 23 460-479 11 11 11 25 25 25 T T T 24 480-499 14 14 14 34 34 34 T T T 25 500-519 13 13 13 31 31 31 F F F 26 520-539 13 13 13 31 31 31 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 6 6 6 12 12 12 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 14, Max flush: 34 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 35 35 35 F F F 1 020-039 14 14 14 33 33 33 F F F 2 040-059 14 14 14 32 32 32 F F F 3 060-079 15 15 15 35 35 35 F F F 4 080-099 14 14 14 33 33 33 F F F 5 100-119 7 7 7 14 14 14 F F F 6 120-139 8 8 8 18 18 18 F F F 7 140-159 14 14 14 32 32 32 F F F 8 160-179 15 15 15 35 35 35 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 4 4 4 7 7 7 F F F 11 220-239 14 14 14 33 33 33 F F F 12 240-259 14 14 14 33 33 33 F F F 13 260-279 12 12 12 27 27 27 F F F 14 280-299 10 10 10 21 21 21 F F F 15 300-319 12 12 12 28 28 28 F F F 16 320-339 11 11 11 25 25 25 F F F 17 340-359 11 11 11 25 25 25 F F F 18 360-379 14 14 14 32 32 32 F F F 19 380-399 9 9 9 19 19 19 F F F 20 400-419 12 12 12 30 30 30 F F F 21 420-439 11 11 11 25 25 25 F F F 22 440-459 11 11 11 25 25 25 F F F 23 460-479 15 15 15 35 35 35 F F F 24 480-499 15 15 15 35 35 35 F F F 25 500-519 13 13 13 31 31 31 F F F 26 520-539 13 13 13 31 31 31 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 6 6 6 12 12 12 F F F 29 580-599 6 6 6 13 13 13 F F F Max gen: 15, Max flush: 35 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.440815442Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.484722637Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.48540972Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.485461085Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.485483183Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.485501571Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251} {"msg":"Crucible ca9c2e62-e0dc-4407-bfc4-dda4e133e192 has session id: 01bb1a3a-1922-494c-8d76-c0362bd97c22","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.486018141Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Upstairs opts: Upstairs UUID: ca9c2e62-e0dc-4407-bfc4-dda4e133e192, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.486061847Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Crucible stats registered with UUID: ca9c2e62-e0dc-4407-bfc4-dda4e133e192","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.486084434Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:36.486104943Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.486507793Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.48654372Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251} {"msg":"ca9c2e62-e0dc-4407-bfc4-dda4e133e192 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.486599014Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.486632271Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"0","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.486671457Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"0","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.486701134Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"1","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.486729232Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"1","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.486754819Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"2","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.486784546Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"2","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.486817163Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.48684601Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.487028562Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"ds_connection connected from Ok(127.0.0.1:38197)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.487064639Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"io task","client":"2","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"ds_connection connected from Ok(127.0.0.1:45647)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.487095226Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"io task","client":"1","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"ds_connection connected from Ok(127.0.0.1:38465)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.487120024Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"io task","client":"0","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.487640333Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"0","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.487666271Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"2","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.487680859Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"1","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.488275241Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"1","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.488418677Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"0","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.488439585Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"2","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.488857044Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.488986262Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"[0]R flush_numbers[0..12]: [35, 33, 32, 35, 33, 14, 18, 32, 35, 25, 7, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489018479Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"[0]R generation[0..12]: [15, 14, 14, 15, 14, 7, 8, 14, 15, 11, 4, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489039077Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489055015Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"[1]R flush_numbers[0..12]: [35, 33, 32, 35, 33, 14, 18, 32, 35, 25, 7, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489068674Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"[1]R generation[0..12]: [15, 14, 14, 15, 14, 7, 8, 14, 15, 11, 4, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489081983Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489095501Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"[2]R flush_numbers[0..12]: [35, 33, 32, 35, 33, 14, 18, 32, 35, 25, 7, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.48910881Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"[2]R generation[0..12]: [15, 14, 14, 15, 14, 7, 8, 14, 15, 11, 4, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489122109Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489135707Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489148446Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489161565Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489174174Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489197901Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"":"downstairs","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.48921224Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489224289Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"0","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489240437Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"1","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489253436Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"client":"2","":"downstairs","session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489266435Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489278274Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"ca9c2e62-e0dc-4407-bfc4-dda4e133e192 is now active with session: 01bb1a3a-1922-494c-8d76-c0362bd97c22","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489290862Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.489302531Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251,"session_id":"01bb1a3a-1922-494c-8d76-c0362bd97c22"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:36.48931444Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 65 len 4096 data: 3 02/30 Write block 576 len 4096 data: 2 03/30 Write block 570 len 16384 data: 2 2 2 2 04/30 Write block 158 len 24576 data: 4 4 4 3 3 3 05/30 Write block 51 len 24576 data: 2 3 3 3 3 3 06/30 Read block 264 len 40960 07/30 Write block 442 len 28672 data: 2 2 2 3 2 2 2 08/30 Read block 238 len 28672 09/30 Read block 244 len 32768 10/30 Read block 201 len 4096 11/30 Read block 81 len 4096 12/30 Write block 268 len 12288 data: 5 5 4 13/30 Read block 242 len 36864 14/30 Read block 580 len 24576 15/30 Flush 16/30 Write block 8 len 4096 data: 3 17/30 Write block 322 len 4096 data: 2 18/30 Read block 281 len 16384 19/30 Write block 579 len 28672 data: 4 4 4 4 4 4 5 20/30 Flush 21/30 Write block 455 len 36864 data: 5 4 4 3 3 3 2 3 3 22/30 Read block 546 len 40960 23/30 Read block 21 len 16384 24/30 Read block 7 len 8192 25/30 Write block 491 len 4096 data: 3 26/30 Write block 219 len 4096 data: 2 27/30 Read block 344 len 12288 28/30 Write block 95 len 16384 data: 2 2 2 2 29/30 Read block 410 len 24576 30/30 Write block 27 len 20480 data: 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:36.507316577Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1251} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 37 37 37 F F F 1 020-039 14 14 14 33 33 33 T T T 2 040-059 16 16 16 36 36 36 F F F 3 060-079 16 16 16 36 36 36 F F F 4 080-099 14 14 14 33 33 33 T T T 5 100-119 7 7 7 14 14 14 F F F 6 120-139 8 8 8 18 18 18 F F F 7 140-159 16 16 16 36 36 36 F F F 8 160-179 16 16 16 36 36 36 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 4 4 4 7 7 7 T T T 11 220-239 14 14 14 33 33 33 F F F 12 240-259 14 14 14 33 33 33 F F F 13 260-279 16 16 16 36 36 36 F F F 14 280-299 10 10 10 21 21 21 F F F 15 300-319 12 12 12 28 28 28 F F F 16 320-339 16 16 16 37 37 37 F F F 17 340-359 11 11 11 25 25 25 F F F 18 360-379 14 14 14 32 32 32 F F F 19 380-399 9 9 9 19 19 19 F F F 20 400-419 12 12 12 30 30 30 F F F 21 420-439 11 11 11 25 25 25 F F F 22 440-459 16 16 16 36 36 36 T T T 23 460-479 15 15 15 35 35 35 T T T 24 480-499 15 15 15 35 35 35 T T T 25 500-519 13 13 13 31 31 31 F F F 26 520-539 13 13 13 31 31 31 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 16 16 16 37 37 37 F F F 29 580-599 16 16 16 37 37 37 F F F Max gen: 16, Max flush: 37 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 37 37 37 F F F 1 020-039 17 17 17 38 38 38 F F F 2 040-059 16 16 16 36 36 36 F F F 3 060-079 16 16 16 36 36 36 F F F 4 080-099 17 17 17 38 38 38 F F F 5 100-119 7 7 7 14 14 14 F F F 6 120-139 8 8 8 18 18 18 F F F 7 140-159 16 16 16 36 36 36 F F F 8 160-179 16 16 16 36 36 36 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 17 17 17 38 38 38 F F F 11 220-239 14 14 14 33 33 33 F F F 12 240-259 14 14 14 33 33 33 F F F 13 260-279 16 16 16 36 36 36 F F F 14 280-299 10 10 10 21 21 21 F F F 15 300-319 12 12 12 28 28 28 F F F 16 320-339 16 16 16 37 37 37 F F F 17 340-359 11 11 11 25 25 25 F F F 18 360-379 14 14 14 32 32 32 F F F 19 380-399 9 9 9 19 19 19 F F F 20 400-419 12 12 12 30 30 30 F F F 21 420-439 11 11 11 25 25 25 F F F 22 440-459 17 17 17 38 38 38 F F F 23 460-479 17 17 17 38 38 38 F F F 24 480-499 17 17 17 38 38 38 F F F 25 500-519 13 13 13 31 31 31 F F F 26 520-539 13 13 13 31 31 31 F F F 27 540-559 10 10 10 21 21 21 F F F 28 560-579 16 16 16 37 37 37 F F F 29 580-599 16 16 16 37 37 37 F F F Max gen: 17, Max flush: 38 Begin loop 9 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.744141436Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:40.785856764Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.786246916Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.817191042Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.817770036Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.817810622Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.81782744Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.817851488Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259} {"msg":"Crucible 08fcca49-d13c-448c-90aa-fde7539e3324 has session id: 5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.818396435Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Upstairs opts: Upstairs UUID: 08fcca49-d13c-448c-90aa-fde7539e3324, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.81845172Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Crucible stats registered with UUID: 08fcca49-d13c-448c-90aa-fde7539e3324","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.818474057Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:40.818490236Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.818907415Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.818937312Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259} {"msg":"08fcca49-d13c-448c-90aa-fde7539e3324 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.819008045Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.819034603Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"0","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.81906134Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"0","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.819090817Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"1","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.819124754Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"1","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.819157811Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"2","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.819192118Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"2","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.819220435Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.819248012Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.81927527Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"ds_connection connected from Ok(127.0.0.1:52910)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.819302487Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"io task","client":"2","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55469)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.819329264Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"io task","client":"1","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47139)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.819393998Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"io task","client":"0","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.820037095Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"0","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.820066433Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"2","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.820082321Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"1","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.820644276Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"1","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.820678673Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"2","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.820698951Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"0","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821229189Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821331989Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"[0]R flush_numbers[0..12]: [37, 38, 36, 36, 38, 14, 18, 36, 36, 25, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821357477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"[0]R generation[0..12]: [16, 17, 16, 16, 17, 7, 8, 16, 16, 11, 17, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821372305Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821386904Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"[1]R flush_numbers[0..12]: [37, 38, 36, 36, 38, 14, 18, 36, 36, 25, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821400633Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"[1]R generation[0..12]: [16, 17, 16, 16, 17, 7, 8, 16, 16, 11, 17, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821414091Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.82142803Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"[2]R flush_numbers[0..12]: [37, 38, 36, 36, 38, 14, 18, 36, 36, 25, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821441389Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"[2]R generation[0..12]: [16, 17, 16, 16, 17, 7, 8, 16, 16, 11, 17, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821454517Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821471466Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821484464Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:09:40.821497473Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821514192Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.82152693Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"":"downstairs","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821540469Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821552378Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"0","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821565407Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"1","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821578175Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"client":"2","":"downstairs","session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821591064Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821603043Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"08fcca49-d13c-448c-90aa-fde7539e3324 is now active with session: 5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821615492Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821627141Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259,"session_id":"5beadd8e-8d0f-4bd6-ad53-9b67a4b9487a"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:40.821639149Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 493 len 28672 data: 3 3 3 2 2 2 2 03/30 Write block 60 len 28672 data: 3 3 2 2 2 4 4 04/30 Write block 61 len 36864 data: 4 3 3 3 5 5 4 4 4 05/30 Write block 112 len 36864 data: 2 2 2 2 2 2 2 2 2 06/30 Write block 536 len 32768 data: 2 2 2 2 2 2 2 2 07/30 Flush 08/30 Write block 256 len 4096 data: 4 09/30 Read block 23 len 12288 10/30 Read block 372 len 32768 11/30 Flush 12/30 Write block 140 len 32768 data: 4 3 3 3 2 2 2 2 13/30 Read block 393 len 24576 14/30 Read block 110 len 40960 15/30 Read block 141 len 8192 16/30 Write block 155 len 40960 data: 3 3 4 5 5 5 4 4 4 4 17/30 Read block 306 len 40960 18/30 Read block 287 len 4096 19/30 Read block 397 len 40960 20/30 Read block 41 len 12288 21/30 Read block 138 len 4096 22/30 Write block 161 len 20480 data: 5 5 5 5 3 23/30 Read block 230 len 20480 24/30 Read block 269 len 36864 25/30 Read block 519 len 16384 26/30 Read block 109 len 12288 27/30 Read block 483 len 28672 28/30 Write block 429 len 40960 data: 2 2 2 2 3 4 3 3 3 3 29/30 Write block 73 len 16384 data: 2 2 2 2 30/30 Read block 438 len 20480 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 8 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 18 19 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:40.842185339Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1259} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 37 37 37 F F F 1 020-039 17 17 17 38 38 38 F F F 2 040-059 16 16 16 36 36 36 F F F 3 060-079 18 18 18 40 40 40 T T T 4 080-099 17 17 17 38 38 38 F F F 5 100-119 18 18 18 40 40 40 F F F 6 120-139 18 18 18 40 40 40 F F F 7 140-159 16 16 16 36 36 36 T T T 8 160-179 16 16 16 36 36 36 T T T 9 180-199 11 11 11 25 25 25 F F F 10 200-219 17 17 17 38 38 38 F F F 11 220-239 14 14 14 33 33 33 F F F 12 240-259 18 18 18 41 41 41 F F F 13 260-279 16 16 16 36 36 36 F F F 14 280-299 10 10 10 21 21 21 F F F 15 300-319 12 12 12 28 28 28 F F F 16 320-339 16 16 16 37 37 37 F F F 17 340-359 11 11 11 25 25 25 F F F 18 360-379 14 14 14 32 32 32 F F F 19 380-399 9 9 9 19 19 19 F F F 20 400-419 12 12 12 30 30 30 F F F 21 420-439 11 11 11 25 25 25 T T T 22 440-459 17 17 17 38 38 38 F F F 23 460-479 17 17 17 38 38 38 F F F 24 480-499 18 18 18 40 40 40 F F F 25 500-519 13 13 13 31 31 31 F F F 26 520-539 18 18 18 40 40 40 F F F 27 540-559 18 18 18 40 40 40 F F F 28 560-579 16 16 16 37 37 37 F F F 29 580-599 16 16 16 37 37 37 F F F Max gen: 18, Max flush: 41 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 37 37 37 F F F 1 020-039 17 17 17 38 38 38 F F F 2 040-059 16 16 16 36 36 36 F F F 3 060-079 19 19 19 42 42 42 F F F 4 080-099 17 17 17 38 38 38 F F F 5 100-119 18 18 18 40 40 40 F F F 6 120-139 18 18 18 40 40 40 F F F 7 140-159 19 19 19 42 42 42 F F F 8 160-179 19 19 19 42 42 42 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 17 17 17 38 38 38 F F F 11 220-239 14 14 14 33 33 33 F F F 12 240-259 18 18 18 41 41 41 F F F 13 260-279 16 16 16 36 36 36 F F F 14 280-299 10 10 10 21 21 21 F F F 15 300-319 12 12 12 28 28 28 F F F 16 320-339 16 16 16 37 37 37 F F F 17 340-359 11 11 11 25 25 25 F F F 18 360-379 14 14 14 32 32 32 F F F 19 380-399 9 9 9 19 19 19 F F F 20 400-419 12 12 12 30 30 30 F F F 21 420-439 19 19 19 42 42 42 F F F 22 440-459 17 17 17 38 38 38 F F F 23 460-479 17 17 17 38 38 38 F F F 24 480-499 18 18 18 40 40 40 F F F 25 500-519 13 13 13 31 31 31 F F F 26 520-539 18 18 18 40 40 40 F F F 27 540-559 18 18 18 40 40 40 F F F 28 560-579 16 16 16 37 37 37 F F F 29 580-599 16 16 16 37 37 37 F F F Max gen: 19, Max flush: 42 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.106330287Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.150324443Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.151024335Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.151065901Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.15108101Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.151097918Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267} {"msg":"Crucible 9f59c02a-2aaa-4479-8ee6-883d3b885073 has session id: c38b647d-f52c-4578-8e00-dd2994f16c3f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.151601269Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"Upstairs opts: Upstairs UUID: 9f59c02a-2aaa-4479-8ee6-883d3b885073, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.151682011Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"Crucible stats registered with UUID: 9f59c02a-2aaa-4479-8ee6-883d3b885073","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.151714088Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:45.151733276Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152151925Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152176303Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267} {"msg":"9f59c02a-2aaa-4479-8ee6-883d3b885073 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152258385Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152276723Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"0","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152299811Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"0","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152313579Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"1","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152327638Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"1","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152340717Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"2","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152354626Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"2","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152367554Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152380973Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152397431Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57360)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152539408Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"io task","client":"1","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45771)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152582803Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"io task","client":"2","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44506)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.152724979Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"io task","client":"0","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.153207912Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"0","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.153246219Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"2","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.153766248Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"2","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.153893586Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"0","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.154241532Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"1","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.154456741Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.154484178Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.154729774Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"1","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"[0]R flush_numbers[0..12]: [37, 38, 36, 42, 38, 40, 40, 42, 42, 25, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.15518662Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"[0]R generation[0..12]: [16, 17, 16, 19, 17, 18, 18, 19, 19, 11, 17, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155208518Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155223476Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"[1]R flush_numbers[0..12]: [37, 38, 36, 42, 38, 40, 40, 42, 42, 25, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155241394Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"[1]R generation[0..12]: [16, 17, 16, 19, 17, 18, 18, 19, 19, 11, 17, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155255383Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155269512Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"[2]R flush_numbers[0..12]: [37, 38, 36, 42, 38, 40, 40, 42, 42, 25, 38, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.15528332Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"[2]R generation[0..12]: [16, 17, 16, 19, 17, 18, 18, 19, 19, 11, 17, 14]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155296719Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155310308Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"Max found gen is 20","v":0,Wait for a query_work_queue command to finish before sending IO "name":"crucible","level":30,"time":"2025-01-28T17:09:45.155329536Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155342855Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155355563Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155368342Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"":"downstairs","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155381561Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.15539337Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"0","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155406408Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"1","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155419317Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"client":"2","":"downstairs","session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155434196Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155446365Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"9f59c02a-2aaa-4479-8ee6-883d3b885073 is now active with session: c38b647d-f52c-4578-8e00-dd2994f16c3f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155458933Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.155474162Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267,"session_id":"c38b647d-f52c-4578-8e00-dd2994f16c3f"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:45.15548673Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 235 len 4096 data: 3 02/30 Flush 03/30 Write block 584 len 4096 data: 5 04/30 Write block 260 len 28672 data: 2 2 3 3 4 5 5 05/30 Write block 81 len 20480 data: 3 3 3 3 3 06/30 Read block 571 len 8192 07/30 Read block 143 len 36864 08/30 Write block 437 len 28672 data: 4 4 3 3 2 3 3 09/30 Write block 518 len 12288 data: 3 3 4 10/30 Read block 504 len 20480 11/30 Read block 237 len 16384 12/30 Write block 274 len 36864 data: 2 2 2 2 3 3 3 3 3 13/30 Read block 245 len 8192 14/30 Write block 436 len 4096 data: 4 15/30 Write block 148 len 16384 data: 2 2 2 2 16/30 Read block 354 len 12288 17/30 Flush 18/30 Write block 358 len 12288 data: 4 3 3 19/30 Read block 200 len 16384 20/30 Read block 584 len 24576 21/30 Write block 136 len 24576 data: 5 5 5 5 5 4 22/30 Read block 220 len 4096 23/30 Read block 308 len 4096 24/30 Write block 211 len 8192 data: 2 2 25/30 Write block 476 len 20480 data: 2 3 4 4 4 26/30 Write block 238 len 28672 data: 3 4 4 4 4 3 3 27/30 Write block 381 len 40960 data: 4 3 3 3 2 2 2 2 2 2 28/30 Read block 347 len 4096 29/30 Write block 161 len 24576 data: 6 6 6 6 4 3 30/30 Read block 556 len 16384 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 13 12 11 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:45.173593707Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1267} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 37 37 37 F F F 1 020-039 17 17 17 38 38 38 F F F 2 040-059 16 16 16 36 36 36 F F F 3 060-079 19 19 19 42 42 42 F F F 4 080-099 20 20 20 44 44 44 F F F 5 100-119 18 18 18 40 40 40 F F F 6 120-139 18 18 18 40 40 40 T T T 7 140-159 20 20 20 44 44 44 T T T 8 160-179 19 19 19 42 42 42 T T T 9 180-199 11 11 11 25 25 25 F F F 10 200-219 17 17 17 38 38 38 T T T 11 220-239 20 20 20 43 43 43 T T T 12 240-259 18 18 18 41 41 41 T T T 13 260-279 20 20 20 44 44 44 F F F 14 280-299 20 20 20 44 44 44 F F F 15 300-319 12 12 12 28 28 28 F F F 16 320-339 16 16 16 37 37 37 F F F 17 340-359 11 11 11 25 25 25 T T T 18 360-379 14 14 14 32 32 32 T T T 19 380-399 9 9 9 19 19 19 T T T 20 400-419 12 12 12 30 30 30 F F F 21 420-439 20 20 20 44 44 44 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 17 17 17 38 38 38 T T T 24 480-499 18 18 18 40 40 40 T T T 25 500-519 20 20 20 44 44 44 F F F 26 520-539 20 20 20 44 44 44 F F F 27 540-559 18 18 18 40 40 40 F F F 28 560-579 16 16 16 37 37 37 F F F 29 580-599 20 20 20 44 44 44 F F F Max gen: 20, Max flush: 44 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 37 37 37 F F F 1 020-039 17 17 17 38 38 38 F F F 2 040-059 16 16 16 36 36 36 F F F 3 060-079 19 19 19 42 42 42 F F F 4 080-099 20 20 20 44 44 44 F F F 5 100-119 18 18 18 40 40 40 F F F 6 120-139 21 21 21 45 45 45 F F F 7 140-159 21 21 21 45 45 45 F F F 8 160-179 21 21 21 45 45 45 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 21 21 21 45 45 45 F F F 11 220-239 21 21 21 45 45 45 F F F 12 240-259 21 21 21 45 45 45 F F F 13 260-279 20 20 20 44 44 44 F F F 14 280-299 20 20 20 44 44 44 F F F 15 300-319 12 12 12 28 28 28 F F F 16 320-339 16 16 16 37 37 37 F F F 17 340-359 21 21 21 45 45 45 F F F 18 360-379 21 21 21 45 45 45 F F F 19 380-399 21 21 21 45 45 45 F F F 20 400-419 12 12 12 30 30 30 F F F 21 420-439 20 20 20 44 44 44 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 21 21 21 45 45 45 F F F 24 480-499 21 21 21 45 45 45 F F F 25 500-519 20 20 20 44 44 44 F F F 26 520-539 20 20 20 44 44 44 F F F 27 540-559 18 18 18 40 40 40 F F F 28 560-579 16 16 16 37 37 37 F F F 29 580-599 20 20 20 44 44 44 F F F Max gen: 21, Max flush: 45 Begin loop 11 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.436459213Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.479119609Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.479780814Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.479820641Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.479835389Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.479849598Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275} {"msg":"Crucible 6a2ffcae-5508-4fca-b64f-82b037192680 has session id: 327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.48034239Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Upstairs opts: Upstairs UUID: 6a2ffcae-5508-4fca-b64f-82b037192680, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.480409633Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Crucible stats registered with UUID: 6a2ffcae-5508-4fca-b64f-82b037192680","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.480432961Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:59.480454249Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.48095071Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.480973368Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275} {"msg":"6a2ffcae-5508-4fca-b64f-82b037192680 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.481041871Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.48105992Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"0","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.481083478Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"0","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.481097926Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"1","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.481116464Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"1","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.481136112Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"2","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.48115975Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"2","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.481183148Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.48126499Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.481300526Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:44762)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.481329023Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"io task","client":"2","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:45239)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.481356781Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"io task","client":"0","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:63524)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.481437633Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"io task","client":"1","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.482045204Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"2","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.48208205Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"0","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.482626297Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"1","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.482665473Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"0","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.482738836Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"2","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483248257Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"1","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483282103Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483296232Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"[0]R flush_numbers[0..12]: [37, 38, 36, 42, 44, 40, 45, 45, 45, 25, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483795933Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"[0]R generation[0..12]: [16, 17, 16, 19, 20, 18, 21, 21, 21, 11, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483823781Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483839009Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"[1]R flush_numbers[0..12]: [37, 38, 36, 42, 44, 40, 45, 45, 45, 25, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483852668Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"[1]R generation[0..12]: [16, 17, 16, 19, 20, 18, 21, 21, 21, 11, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483866036Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483879785Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"[2]R flush_numbers[0..12]: [37, 38, 36, 42, 44, 40, 45, 45, 45, 25, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483893274Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"[2]R generation[0..12]: [16, 17, 16, 19, 20, 18, 21, 21, 21, 11, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483907103Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483921151Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:09:59.48393401Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483949438Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483965737Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483978816Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"":"downstairs","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.483991654Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.484003593Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"0","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.484016612Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"1","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.484029231Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"client":"2","":"downstairs","session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.484042089Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.484054248Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"6a2ffcae-5508-4fca-b64f-82b037192680 is now active with session: 327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.484066837Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"session_id":"327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.484078536Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 327ec5ea-2828-4aff-aaa5-f9a4e9cea2cc"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:09:59.484094314Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 102 len 32768 data: 2 2 2 2 2 3 4 4 02/30 Read block 273 len 36864 03/30 Read block 370 len 8192 04/30 Read block 227 len 40960 05/30 Write block 509 len 32768 data: 2 3 3 4 5 4 4 4 06/30 Read block 241 len 28672 07/30 Read block 197 len 4096 08/30 Write block 55 len 4096 data: 4 09/30 Write block 289 len 24576 data: 2 2 2 2 2 2 10/30 Read block 250 len 20480 11/30 Read block 424 len 8192 12/30 Read block 164 len 12288 13/30 Write block 568 len 12288 data: 2 2 3 14/30 Flush 15/30 Write block 410 len 40960 data: 2 2 2 2 2 2 3 3 2 2 16/30 Read block 541 len 24576 17/30 Write block 220 len 28672 data: 2 2 2 2 2 3 4 18/30 Write block 6 len 32768 data: 3 4 4 2 2 2 2 2 19/30 Write block 297 len 24576 data: 2 2 2 3 3 3 20/30 Read block 167 len 20480 21/30 Read block 442 len 12288 22/30 Read block 426 len 20480 23/30 Read block 446 len 24576 24/30 Read block 507 len 20480 25/30 Write block 34 len 12288 data: 2 2 2 26/30 Write block 420 len 32768 data: 2 2 2 2 2 2 2 2 27/30 Read block 524 len 4096 28/30 Write block 323 len 24576 data: 2 2 2 2 2 2 29/30 Read block 211 len 12288 30/30 Read block 333 len 16384 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 10 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 16 15 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:09:59.504349612Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1275} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 37 37 37 T T T 1 020-039 17 17 17 38 38 38 T T T 2 040-059 22 22 22 46 46 46 F F F 3 060-079 19 19 19 42 42 42 F F F 4 080-099 20 20 20 44 44 44 F F F 5 100-119 22 22 22 46 46 46 F F F 6 120-139 21 21 21 45 45 45 F F F 7 140-159 21 21 21 45 45 45 F F F 8 160-179 21 21 21 45 45 45 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 21 21 21 45 45 45 F F F 11 220-239 21 21 21 45 45 45 T T T 12 240-259 21 21 21 45 45 45 F F F 13 260-279 20 20 20 44 44 44 F F F 14 280-299 22 22 22 46 46 46 T T T 15 300-319 12 12 12 28 28 28 T T T 16 320-339 16 16 16 37 37 37 T T T 17 340-359 21 21 21 45 45 45 F F F 18 360-379 21 21 21 45 45 45 F F F 19 380-399 21 21 21 45 45 45 F F F 20 400-419 12 12 12 30 30 30 T T T 21 420-439 20 20 20 44 44 44 T T T 22 440-459 20 20 20 44 44 44 F F F 23 460-479 21 21 21 45 45 45 F F F 24 480-499 21 21 21 45 45 45 F F F 25 500-519 22 22 22 46 46 46 F F F 26 520-539 20 20 20 44 44 44 F F F 27 540-559 18 18 18 40 40 40 F F F 28 560-579 22 22 22 46 46 46 F F F 29 580-599 20 20 20 44 44 44 F F F Max gen: 22, Max flush: 46 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 47 47 47 F F F 1 020-039 23 23 23 47 47 47 F F F 2 040-059 22 22 22 46 46 46 F F F 3 060-079 19 19 19 42 42 42 F F F 4 080-099 20 20 20 44 44 44 F F F 5 100-119 22 22 22 46 46 46 F F F 6 120-139 21 21 21 45 45 45 F F F 7 140-159 21 21 21 45 45 45 F F F 8 160-179 21 21 21 45 45 45 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 21 21 21 45 45 45 F F F 11 220-239 23 23 23 47 47 47 F F F 12 240-259 21 21 21 45 45 45 F F F 13 260-279 20 20 20 44 44 44 F F F 14 280-299 23 23 23 47 47 47 F F F 15 300-319 23 23 23 47 47 47 F F F 16 320-339 23 23 23 47 47 47 F F F 17 340-359 21 21 21 45 45 45 F F F 18 360-379 21 21 21 45 45 45 F F F 19 380-399 21 21 21 45 45 45 F F F 20 400-419 23 23 23 47 47 47 F F F 21 420-439 23 23 23 47 47 47 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 21 21 21 45 45 45 F F F 24 480-499 21 21 21 45 45 45 F F F 25 500-519 22 22 22 46 46 46 F F F 26 520-539 20 20 20 44 44 44 F F F 27 540-559 18 18 18 40 40 40 F F F 28 560-579 22 22 22 46 46 46 F F F 29 580-599 20 20 20 44 44 44 F F F Max gen: 23, Max flush: 47 Begin loop 12 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.748797148Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:03.790083427Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.79046584Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.821154752Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.821720476Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.821755083Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.821769381Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.82178357Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283} {"msg":"Crucible d0e4fb0b-d17b-4745-829e-afcfad5dac0d has session id: 39d1901d-a96b-4349-98f2-3d68f3e2f6a2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.82240021Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Upstairs opts: Upstairs UUID: d0e4fb0b-d17b-4745-829e-afcfad5dac0d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.822439596Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Crucible stats registered with UUID: d0e4fb0b-d17b-4745-829e-afcfad5dac0d","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.822460204Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:03.822474073Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.822867605Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.822890732Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283} {"msg":"d0e4fb0b-d17b-4745-829e-afcfad5dac0d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.822946127Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.822973584Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"0","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.822998392Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"0","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.82301282Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"1","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.823027279Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"1","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.823040508Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"2","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.823054667Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"2","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.823067665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.823085853Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.823110361Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:38671)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.823135399Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"io task","client":"2","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:47831)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.823181014Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"io task","client":"1","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:40036)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.823213371Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"io task","client":"0","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.82394784Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"1","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.824019333Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"0","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.824059229Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"2","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.824520184Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"1","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.824549131Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"0","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.82466495Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"2","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825036243Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825119745Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"[0]R flush_numbers[0..12]: [47, 47, 46, 42, 44, 46, 45, 45, 45, 25, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825233804Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"[0]R generation[0..12]: [23, 23, 22, 19, 20, 22, 21, 21, 21, 11, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825253192Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825267591Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"[1]R flush_numbers[0..12]: [47, 47, 46, 42, 44, 46, 45, 45, 45, 25, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825281659Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"[1]R generation[0..12]: [23, 23, 22, 19, 20, 22, 21, 21, 21, 11, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825299448Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825313767Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"[2]R flush_numbers[0..12]: [47, 47, 46, 42, 44, 46, 45, 45, 45, 25, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825327245Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"[2]R generation[0..12]: [23, 23, 22, 19, 20, 22, 21, 21, 21, 11, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825340594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825354123Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825366561Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.82538516Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825397898Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825410507Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"":"downstairs","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825423656Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825435305Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"0","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825448643Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"1","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825461592Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"client":"2","":"downstairs","session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825474561Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825493039Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"d0e4fb0b-d17b-4745-829e-afcfad5dac0d is now active with session: 39d1901d-a96b-4349-98f2-3d68f3e2f6a2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825510547Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825527046Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283,"session_id":"39d1901d-a96b-4349-98f2-3d68f3e2f6a2"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:03.825539074Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 492 len 40960 data: 3 4 4 4 3 3 3 3 3 3 02/30 Read block 381 len 40960 03/30 Read block 66 len 8192 04/30 Write block 349 len 28672 data: 4 4 4 4 5 4 3 05/30 Write block 322 len 32768 data: 3 3 3 3 3 3 3 2 06/30 Read block 530 len 12288 07/30 Read block 297 len 24576 08/30 Write block 477 len 12288 data: 4 5 5 09/30 Read block 325 len 12288 10/30 Read block 316 len 16384 11/30 Write block 252 len 32768 data: 5 5 5 4 5 4 3 2 12/30 Write block 25 len 40960 data: 3 3 3 3 3 3 3 2 2 3 13/30 Read block 404 len 16384 14/30 Read block 17 len 8192 15/30 Write block 22 len 20480 data: 3 3 3 4 4 16/30 Write block 551 len 40960 data: 3 3 3 3 2 2 2 2 2 2 17/30 Read block 371 len 28672 18/30 Read block 152 len 24576 19/30 Read block 442 len 12288 20/30 Read block 55 len 28672 21/30 Read block 191 len 16384 22/30 Flush 23/30 Write block 85 len 12288 data: 4 2 2 24/30 Read block 587 len 16384 25/30 Write block 523 len 12288 data: 4 5 4 26/30 Read block 120 len 12288 27/30 Read block 126 len 4096 28/30 Write block 398 len 36864 data: 2 2 4 4 4 5 3 4 4 29/30 Write block 469 len 20480 data: 3 3 3 3 3 30/30 Read block 200 len 8192 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 8 6 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:03.846047968Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1283} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 47 47 47 F F F 1 020-039 24 24 24 48 48 48 F F F 2 040-059 22 22 22 46 46 46 F F F 3 060-079 19 19 19 42 42 42 F F F 4 080-099 20 20 20 44 44 44 T T T 5 100-119 22 22 22 46 46 46 F F F 6 120-139 21 21 21 45 45 45 F F F 7 140-159 21 21 21 45 45 45 F F F 8 160-179 21 21 21 45 45 45 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 21 21 21 45 45 45 F F F 11 220-239 23 23 23 47 47 47 F F F 12 240-259 24 24 24 48 48 48 F F F 13 260-279 20 20 20 44 44 44 F F F 14 280-299 23 23 23 47 47 47 F F F 15 300-319 23 23 23 47 47 47 F F F 16 320-339 24 24 24 48 48 48 F F F 17 340-359 24 24 24 48 48 48 F F F 18 360-379 21 21 21 45 45 45 F F F 19 380-399 21 21 21 45 45 45 T T T 20 400-419 23 23 23 47 47 47 T T T 21 420-439 23 23 23 47 47 47 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 24 24 24 48 48 48 T T T 24 480-499 24 24 24 48 48 48 F F F 25 500-519 24 24 24 48 48 48 F F F 26 520-539 20 20 20 44 44 44 T T T 27 540-559 24 24 24 48 48 48 F F F 28 560-579 24 24 24 48 48 48 F F F 29 580-599 20 20 20 44 44 44 F F F Max gen: 24, Max flush: 48 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 47 47 47 F F F 1 020-039 24 24 24 48 48 48 F F F 2 040-059 22 22 22 46 46 46 F F F 3 060-079 19 19 19 42 42 42 F F F 4 080-099 25 25 25 49 49 49 F F F 5 100-119 22 22 22 46 46 46 F F F 6 120-139 21 21 21 45 45 45 F F F 7 140-159 21 21 21 45 45 45 F F F 8 160-179 21 21 21 45 45 45 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 21 21 21 45 45 45 F F F 11 220-239 23 23 23 47 47 47 F F F 12 240-259 24 24 24 48 48 48 F F F 13 260-279 20 20 20 44 44 44 F F F 14 280-299 23 23 23 47 47 47 F F F 15 300-319 23 23 23 47 47 47 F F F 16 320-339 24 24 24 48 48 48 F F F 17 340-359 24 24 24 48 48 48 F F F 18 360-379 21 21 21 45 45 45 F F F 19 380-399 25 25 25 49 49 49 F F F 20 400-419 25 25 25 49 49 49 F F F 21 420-439 23 23 23 47 47 47 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 25 25 25 49 49 49 F F F 24 480-499 24 24 24 48 48 48 F F F 25 500-519 24 24 24 48 48 48 F F F 26 520-539 25 25 25 49 49 49 F F F 27 540-559 24 24 24 48 48 48 F F F 28 560-579 24 24 24 48 48 48 F F F 29 580-599 20 20 20 44 44 44 F F F Max gen: 25, Max flush: 49 Begin loop 13 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.112764436Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:08.154494942Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.154894563Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.185800424Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.186374628Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.186408455Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.186422034Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.186437222Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291} {"msg":"Crucible 0092e767-7de6-4d1d-aa75-9bd3fa0d89c5 has session id: 7889ccd5-b61d-4650-af76-ac14ea4f0b23","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.187096828Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Upstairs opts: Upstairs UUID: 0092e767-7de6-4d1d-aa75-9bd3fa0d89c5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.187145083Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Crucible stats registered with UUID: 0092e767-7de6-4d1d-aa75-9bd3fa0d89c5","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.187161151Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:08.18717428Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.187708598Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.187741895Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291} {"msg":"0092e767-7de6-4d1d-aa75-9bd3fa0d89c5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.187803819Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.187821977Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"0","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.187843495Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"0","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.187866183Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"1","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.1878979Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"1","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.187925547Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"2","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.187953374Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"2","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.187982602Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.188011989Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.188035426Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"ds_connection connected from Ok(127.0.0.1:33564)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.188061284Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"io task","client":"2","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"ds_connection connected from Ok(127.0.0.1:58900)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.188085761Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"io task","client":"1","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"ds_connection connected from Ok(127.0.0.1:38739)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.188214439Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"io task","client":"0","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.188681423Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"1","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.188710781Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"2","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.188873955Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"0","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.189375136Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"1","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.189403853Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"0","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.189420022Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"2","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.18995101Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190017333Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"[0]R flush_numbers[0..12]: [47, 48, 46, 42, 49, 46, 45, 45, 45, 25, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190075858Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"[0]R generation[0..12]: [23, 24, 22, 19, 25, 22, 21, 21, 21, 11, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190095276Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190109784Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"[1]R flush_numbers[0..12]: [47, 48, 46, 42, 49, 46, 45, 45, 45, 25, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190123463Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"[1]R generation[0..12]: [23, 24, 22, 19, 25, 22, 21, 21, 21, 11, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190136892Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.19015071Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"[2]R flush_numbers[0..12]: [47, 48, 46, 42, 49, 46, 45, 45, 45, 25, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190164299Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"[2]R generation[0..12]: [23, 24, 22, 19, 25, 22, 21, 21, 21, 11, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190177608Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190191267Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190204325Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,Wait for a query_work_queue command to finish before sending IO "":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190225653Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190238432Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.19025441Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"":"downstairs","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190268059Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190279878Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"0","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190292946Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"1","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190305895Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"client":"2","":"downstairs","session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190318924Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190330993Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"0092e767-7de6-4d1d-aa75-9bd3fa0d89c5 is now active with session: 7889ccd5-b61d-4650-af76-ac14ea4f0b23","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190343801Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.19035578Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291,"session_id":"7889ccd5-b61d-4650-af76-ac14ea4f0b23"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:08.190368119Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 308 len 20480 data: 4 5 4 4 3 02/30 Read block 278 len 16384 03/30 Write block 537 len 20480 data: 3 3 3 3 3 04/30 Flush 05/30 Write block 571 len 16384 data: 3 3 3 2 06/30 Read block 538 len 32768 07/30 Read block 335 len 8192 08/30 Read block 587 len 32768 09/30 Read block 429 len 16384 10/30 Read block 434 len 36864 11/30 Write block 95 len 32768 data: 3 3 3 3 2 2 2 3 12/30 Read block 94 len 32768 13/30 Read block 512 len 4096 14/30 Read block 467 len 24576 15/30 Read block 485 len 40960 16/30 Read block 490 len 16384 17/30 Write block 327 len 32768 data: 4 4 3 2 2 2 2 2 18/30 Read block 564 len 32768 19/30 Read block 180 len 12288 20/30 Write block 414 len 12288 data: 3 3 4 21/30 Write block 582 len 4096 data: 5 22/30 Read block 229 len 12288 23/30 Read block 399 len 16384 24/30 Flush 25/30 Write block 369 len 12288 data: 2 2 2 26/30 Write block 71 len 36864 data: 5 4 3 3 3 3 2 2 3 27/30 Read block 69 len 20480 28/30 Write block 9 len 32768 data: 3 3 3 3 3 2 2 3 29/30 Write block 311 len 32768 data: 5 4 3 4 4 3 3 3 30/30 Read block 489 len 4096 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:08.210586661Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1291} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 47 47 47 T T T 1 020-039 24 24 24 48 48 48 F F F 2 040-059 22 22 22 46 46 46 F F F 3 060-079 19 19 19 42 42 42 T T T 4 080-099 26 26 26 51 51 51 F F F 5 100-119 26 26 26 51 51 51 F F F 6 120-139 21 21 21 45 45 45 F F F 7 140-159 21 21 21 45 45 45 F F F 8 160-179 21 21 21 45 45 45 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 21 21 21 45 45 45 F F F 11 220-239 23 23 23 47 47 47 F F F 12 240-259 24 24 24 48 48 48 F F F 13 260-279 20 20 20 44 44 44 F F F 14 280-299 23 23 23 47 47 47 F F F 15 300-319 26 26 26 50 50 50 T T T 16 320-339 26 26 26 51 51 51 F F F 17 340-359 24 24 24 48 48 48 F F F 18 360-379 21 21 21 45 45 45 T T T 19 380-399 25 25 25 49 49 49 F F F 20 400-419 26 26 26 51 51 51 F F F 21 420-439 23 23 23 47 47 47 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 25 25 25 49 49 49 F F F 24 480-499 24 24 24 48 48 48 F F F 25 500-519 24 24 24 48 48 48 F F F 26 520-539 26 26 26 50 50 50 F F F 27 540-559 26 26 26 50 50 50 F F F 28 560-579 26 26 26 51 51 51 F F F 29 580-599 26 26 26 51 51 51 F F F Max gen: 26, Max flush: 51 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 52 52 52 F F F 1 020-039 24 24 24 48 48 48 F F F 2 040-059 22 22 22 46 46 46 F F F 3 060-079 27 27 27 52 52 52 F F F 4 080-099 26 26 26 51 51 51 F F F 5 100-119 26 26 26 51 51 51 F F F 6 120-139 21 21 21 45 45 45 F F F 7 140-159 21 21 21 45 45 45 F F F 8 160-179 21 21 21 45 45 45 F F F 9 180-199 11 11 11 25 25 25 F F F 10 200-219 21 21 21 45 45 45 F F F 11 220-239 23 23 23 47 47 47 F F F 12 240-259 24 24 24 48 48 48 F F F 13 260-279 20 20 20 44 44 44 F F F 14 280-299 23 23 23 47 47 47 F F F 15 300-319 27 27 27 52 52 52 F F F 16 320-339 26 26 26 51 51 51 F F F 17 340-359 24 24 24 48 48 48 F F F 18 360-379 27 27 27 52 52 52 F F F 19 380-399 25 25 25 49 49 49 F F F 20 400-419 26 26 26 51 51 51 F F F 21 420-439 23 23 23 47 47 47 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 25 25 25 49 49 49 F F F 24 480-499 24 24 24 48 48 48 F F F 25 500-519 24 24 24 48 48 48 F F F 26 520-539 26 26 26 50 50 50 F F F 27 540-559 26 26 26 50 50 50 F F F 28 560-579 26 26 26 51 51 51 F F F 29 580-599 26 26 26 51 51 51 F F F Max gen: 27, Max flush: 52 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.467791335Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.511031824Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.511628326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.511669372Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.511684891Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.511699499Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299} {"msg":"Crucible a2cea499-4b8b-4552-9dee-7cbedb5274f9 has session id: 287faea8-b764-462f-8806-2e9399dafcbd","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512216189Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Upstairs opts: Upstairs UUID: a2cea499-4b8b-4552-9dee-7cbedb5274f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512265334Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Crucible stats registered with UUID: a2cea499-4b8b-4552-9dee-7cbedb5274f9","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512288162Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:12.512302261Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512771775Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512795892Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299} {"msg":"a2cea499-4b8b-4552-9dee-7cbedb5274f9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512862116Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512882814Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"0","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512904942Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"0","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.51291904Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"1","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512934839Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"1","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512947858Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"2","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512961806Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"2","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512974645Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.512988044Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.513001232Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:59812)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.51312795Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"io task","client":"1","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:57415)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.513350778Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"io task","client":"0","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:35428)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.513392945Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"io task","client":"2","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.51385287Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"2","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.514024803Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"0","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.514483978Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"2","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.514628304Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"1","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.514662521Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"0","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515090569Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515199578Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.51528277Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"1","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"[0]R flush_numbers[0..12]: [52, 48, 46, 52, 51, 51, 45, 45, 45, 25, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515754404Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"[0]R generation[0..12]: [27, 24, 22, 27, 26, 26, 21, 21, 21, 11, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515777912Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515792781Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"[1]R flush_numbers[0..12]: [52, 48, 46, 52, 51, 51, 45, 45, 45, 25, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515806699Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"[1]R generation[0..12]: [27, 24, 22, 27, 26, 26, 21, 21, 21, 11, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515820728Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515834677Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"[2]R flush_numbers[0..12]: [52, 48, 46, 52, 51, 51, 45, 45, 45, 25, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515848195Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"[2]R generation[0..12]: [27, 24, 22, 27, 26, 26, 21, 21, 21, 11, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515861874Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515880072Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:10:12.515893291Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515921978Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515946846Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515971483Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"":"downstairs","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.515997331Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.516021309Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"0","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.516045696Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"1","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.516069334Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"client":"2","":"downstairs","session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.516094661Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.516118799Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"a2cea499-4b8b-4552-9dee-7cbedb5274f9 is now active with session: 287faea8-b764-462f-8806-2e9399dafcbd","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.516143757Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:12.516168184Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299,"session_id":"287faea8-b764-462f-8806-2e9399dafcbd"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-28T17:10:12.516191912Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 184 len 32768 data: 4 4 3 3 2 3 3 3 02/30 Flush 03/30 Write block 167 len 8192 data: 3 5 04/30 Read block 182 len 40960 05/30 Write block 175 len 4096 data: 5 06/30 Write block 545 len 4096 data: 2 07/30 Write block 321 len 32768 data: 2 4 4 4 4 4 5 5 08/30 Write block 137 len 20480 data: 6 6 6 6 5 09/30 Read block 558 len 12288 10/30 Write block 182 len 8192 data: 5 5 11/30 Read block 381 len 8192 12/30 Write block 480 len 16384 data: 5 4 6 6 13/30 Read block 563 len 4096 14/30 Read block 13 len 8192 15/30 Read block 76 len 8192 16/30 Flush 17/30 Write block 97 len 12288 data: 4 4 3 18/30 Write block 504 len 32768 data: 3 3 2 3 2 3 4 4 19/30 Read block 338 len 24576 20/30 Flush 21/30 Write block 412 len 16384 data: 3 3 4 4 22/30 Write block 62 len 28672 data: 4 4 4 6 6 5 5 23/30 Write block 535 len 20480 data: 2 3 4 4 4 24/30 Read block 286 len 28672 25/30 Read block 257 len 8192 26/30 Read block 318 len 32768 27/30 Write block 562 len 8192 data: 2 2 28/30 Read block 468 len 20480 29/30 Read block 444 len 32768 30/30 Read block 344 len 28672 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 10 9 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:12.534059932Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1299} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 52 52 52 F F F 1 020-039 24 24 24 48 48 48 F F F 2 040-059 22 22 22 46 46 46 F F F 3 060-079 27 27 27 52 52 52 T T T 4 080-099 28 28 28 55 55 55 F F F 5 100-119 26 26 26 51 51 51 F F F 6 120-139 28 28 28 54 54 54 F F F 7 140-159 28 28 28 54 54 54 F F F 8 160-179 28 28 28 54 54 54 F F F 9 180-199 28 28 28 54 54 54 F F F 10 200-219 21 21 21 45 45 45 F F F 11 220-239 23 23 23 47 47 47 F F F 12 240-259 24 24 24 48 48 48 F F F 13 260-279 20 20 20 44 44 44 F F F 14 280-299 23 23 23 47 47 47 F F F 15 300-319 27 27 27 52 52 52 F F F 16 320-339 28 28 28 54 54 54 F F F 17 340-359 24 24 24 48 48 48 F F F 18 360-379 27 27 27 52 52 52 F F F 19 380-399 25 25 25 49 49 49 F F F 20 400-419 26 26 26 51 51 51 T T T 21 420-439 23 23 23 47 47 47 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 25 25 25 49 49 49 F F F 24 480-499 28 28 28 54 54 54 F F F 25 500-519 28 28 28 55 55 55 F F F 26 520-539 26 26 26 50 50 50 T T T 27 540-559 28 28 28 54 54 54 F F F 28 560-579 26 26 26 51 51 51 T T T 29 580-599 26 26 26 51 51 51 F F F Max gen: 28, Max flush: 55 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 52 52 52 F F F 1 020-039 24 24 24 48 48 48 F F F 2 040-059 22 22 22 46 46 46 F F F 3 060-079 29 29 29 56 56 56 F F F 4 080-099 28 28 28 55 55 55 F F F 5 100-119 26 26 26 51 51 51 F F F 6 120-139 28 28 28 54 54 54 F F F 7 140-159 28 28 28 54 54 54 F F F 8 160-179 28 28 28 54 54 54 F F F 9 180-199 28 28 28 54 54 54 F F F 10 200-219 21 21 21 45 45 45 F F F 11 220-239 23 23 23 47 47 47 F F F 12 240-259 24 24 24 48 48 48 F F F 13 260-279 20 20 20 44 44 44 F F F 14 280-299 23 23 23 47 47 47 F F F 15 300-319 27 27 27 52 52 52 F F F 16 320-339 28 28 28 54 54 54 F F F 17 340-359 24 24 24 48 48 48 F F F 18 360-379 27 27 27 52 52 52 F F F 19 380-399 25 25 25 49 49 49 F F F 20 400-419 29 29 29 56 56 56 F F F 21 420-439 23 23 23 47 47 47 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 25 25 25 49 49 49 F F F 24 480-499 28 28 28 54 54 54 F F F 25 500-519 28 28 28 55 55 55 F F F 26 520-539 29 29 29 56 56 56 F F F 27 540-559 28 28 28 54 54 54 F F F 28 560-579 29 29 29 56 56 56 F F F 29 580-599 26 26 26 51 51 51 F F F Max gen: 29, Max flush: 56 Begin loop 15 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.758488597Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:16.799893955Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.800279737Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.830975488Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.831558041Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.831594568Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.831608957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.831624175Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307} {"msg":"Crucible e05c2136-7f02-42c9-9904-92a11e5c4908 has session id: 844109a0-209d-44af-9a8f-dfca5b0199b2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.832257733Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Upstairs opts: Upstairs UUID: e05c2136-7f02-42c9-9904-92a11e5c4908, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.83229319Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Crucible stats registered with UUID: e05c2136-7f02-42c9-9904-92a11e5c4908","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.832310648Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:16.832323347Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.832749156Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.832772113Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307} {"msg":"e05c2136-7f02-42c9-9904-92a11e5c4908 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.832826168Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.832852406Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"0","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.832875733Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"0","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.832890672Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"1","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.832916029Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"1","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.832937567Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"2","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.83300914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"2","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.833046807Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.833075894Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.833184123Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"ds_connection connected from Ok(127.0.0.1:50276)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.83322108Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"io task","client":"2","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43957)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.833248297Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"io task","client":"1","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43503)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.833264215Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"io task","client":"0","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.833866767Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"1","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.833897714Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"2","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.833924811Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"0","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.834539551Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"1","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.834567748Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"0","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.834583927Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"2","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835147732Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835179609Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"[0]R flush_numbers[0..12]: [52, 48, 46, 56, 55, 51, 54, 54, 54, 54, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835209756Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"[0]R generation[0..12]: [27, 24, 22, 29, 28, 26, 28, 28, 28, 28, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835225124Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835242653Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"[1]R flush_numbers[0..12]: [52, 48, 46, 56, 55, 51, 54, 54, 54, 54, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835256641Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"[1]R generation[0..12]: [27, 24, 22, 29, 28, 26, 28, 28, 28, 28, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.83527026Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835284169Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"[2]R flush_numbers[0..12]: [52, 48, 46, 56, 55, 51, 54, 54, 54, 54, 45, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835297497Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"[2]R generation[0..12]: [27, 24, 22, 29, 28, 26, 28, 28, 28, 28, 21, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835311336Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835325125Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835337624Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835350432Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Next flush: 57","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-01-28T17:10:16.83536885Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835381799Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"":"downstairs","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835395208Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835406887Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"0","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835420256Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"1","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835433384Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"client":"2","":"downstairs","session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835448713Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835460841Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"e05c2136-7f02-42c9-9904-92a11e5c4908 is now active with session: 844109a0-209d-44af-9a8f-dfca5b0199b2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.83547341Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835490899Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307,"session_id":"844109a0-209d-44af-9a8f-dfca5b0199b2"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:16.835506007Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 40 len 32768 data: 3 3 3 3 3 2 2 2 02/30 Write block 204 len 12288 data: 3 3 3 03/30 Read block 532 len 40960 04/30 Flush 05/30 Write block 7 len 40960 data: 5 5 4 4 4 4 4 3 3 4 06/30 Write block 248 len 28672 data: 4 3 4 5 6 6 6 07/30 Read block 3 len 24576 08/30 Write block 503 len 32768 data: 3 4 4 3 4 3 4 5 09/30 Read block 541 len 20480 10/30 Read block 397 len 32768 11/30 Read block 190 len 32768 12/30 Write block 403 len 16384 data: 6 4 5 5 13/30 Read block 453 len 4096 14/30 Read block 518 len 20480 15/30 Write block 240 len 40960 data: 5 5 5 4 4 4 4 4 5 4 16/30 Write block 275 len 20480 data: 3 3 3 4 4 17/30 Read block 575 len 28672 18/30 Read block 12 len 40960 19/30 Read block 345 len 28672 20/30 Write block 315 len 16384 data: 5 4 4 4 21/30 Write block 504 len 28672 data: 5 5 4 5 4 5 6 22/30 Read block 211 len 32768 23/30 Flush 24/30 Write block 117 len 28672 data: 3 3 3 3 2 2 2 25/30 Flush 26/30 Write block 383 len 24576 data: 4 4 3 3 3 3 27/30 Read block 37 len 40960 28/30 Read block 463 len 40960 29/30 Write block 583 len 28672 data: 5 6 6 3 2 2 3 30/30 Read block 20 len 20480 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 5 4 3 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:16.85673926Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1307} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 58 58 58 F F F 1 020-039 24 24 24 48 48 48 F F F 2 040-059 30 30 30 57 57 57 F F F 3 060-079 29 29 29 56 56 56 F F F 4 080-099 28 28 28 55 55 55 F F F 5 100-119 30 30 30 59 59 59 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 54 54 54 F F F 8 160-179 28 28 28 54 54 54 F F F 9 180-199 28 28 28 54 54 54 F F F 10 200-219 30 30 30 57 57 57 F F F 11 220-239 23 23 23 47 47 47 F F F 12 240-259 30 30 30 58 58 58 F F F 13 260-279 30 30 30 58 58 58 F F F 14 280-299 23 23 23 47 47 47 F F F 15 300-319 30 30 30 58 58 58 F F F 16 320-339 28 28 28 54 54 54 F F F 17 340-359 24 24 24 48 48 48 F F F 18 360-379 27 27 27 52 52 52 F F F 19 380-399 25 25 25 49 49 49 T T T 20 400-419 30 30 30 58 58 58 F F F 21 420-439 23 23 23 47 47 47 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 25 25 25 49 49 49 F F F 24 480-499 28 28 28 54 54 54 F F F 25 500-519 30 30 30 58 58 58 F F F 26 520-539 29 29 29 56 56 56 F F F 27 540-559 28 28 28 54 54 54 F F F 28 560-579 29 29 29 56 56 56 F F F 29 580-599 26 26 26 51 51 51 T T T Max gen: 30, Max flush: 59 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 58 58 58 F F F 1 020-039 24 24 24 48 48 48 F F F 2 040-059 30 30 30 57 57 57 F F F 3 060-079 29 29 29 56 56 56 F F F 4 080-099 28 28 28 55 55 55 F F F 5 100-119 30 30 30 59 59 59 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 54 54 54 F F F 8 160-179 28 28 28 54 54 54 F F F 9 180-199 28 28 28 54 54 54 F F F 10 200-219 30 30 30 57 57 57 F F F 11 220-239 23 23 23 47 47 47 F F F 12 240-259 30 30 30 58 58 58 F F F 13 260-279 30 30 30 58 58 58 F F F 14 280-299 23 23 23 47 47 47 F F F 15 300-319 30 30 30 58 58 58 F F F 16 320-339 28 28 28 54 54 54 F F F 17 340-359 24 24 24 48 48 48 F F F 18 360-379 27 27 27 52 52 52 F F F 19 380-399 31 31 31 60 60 60 F F F 20 400-419 30 30 30 58 58 58 F F F 21 420-439 23 23 23 47 47 47 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 25 25 25 49 49 49 F F F 24 480-499 28 28 28 54 54 54 F F F 25 500-519 30 30 30 58 58 58 F F F 26 520-539 29 29 29 56 56 56 F F F 27 540-559 28 28 28 54 54 54 F F F 28 560-579 29 29 29 56 56 56 F F F 29 580-599 31 31 31 60 60 60 F F F Max gen: 31, Max flush: 60 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.105935034Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.149046827Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.149647098Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.149689634Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.149708702Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.149723831Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315} {"msg":"Crucible 6cd151f6-a3e5-4cbb-866e-bcfb1017b6cf has session id: 175d52ae-0e6b-495f-bb06-d78a94a4bc60","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.150280216Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Upstairs opts: Upstairs UUID: 6cd151f6-a3e5-4cbb-866e-bcfb1017b6cf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.150336171Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Crucible stats registered with UUID: 6cd151f6-a3e5-4cbb-866e-bcfb1017b6cf","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.150358699Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:21.150372458Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.150818944Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.150842562Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315} {"msg":"6cd151f6-a3e5-4cbb-866e-bcfb1017b6cf active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.150904456Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.150922604Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"0","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.150943832Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"0","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.150957541Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"1","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.150971489Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"1","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.150987878Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"2","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.151002116Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"2","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.151014825Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.151029204Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.151052571Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"ds_connection connected from Ok(127.0.0.1:64535)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.151200497Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"io task","client":"0","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"ds_connection connected from Ok(127.0.0.1:48496)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.151240803Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"io task","client":"2","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"ds_connection connected from Ok(127.0.0.1:56049)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.15126752Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"io task","client":"1","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.151838955Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"2","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.151875021Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"0","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.151904038Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"1","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.152522808Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"2","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.152651696Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"0","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.152680603Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"1","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153125159Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153243388Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"[0]R flush_numbers[0..12]: [58, 48, 57, 56, 55, 59, 59, 54, 54, 54, 57, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.15332445Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"[0]R generation[0..12]: [30, 24, 30, 29, 28, 30, 30, 28, 28, 28, 30, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153344828Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153359257Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"[1]R flush_numbers[0..12]: [58, 48, 57, 56, 55, 59, 59, 54, 54, 54, 57, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153372845Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"[1]R generation[0..12]: [30, 24, 30, 29, 28, 30, 30, 28, 28, 28, 30, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153386454Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153401073Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"[2]R flush_numbers[0..12]: [58, 48, 57, 56, 55, 59, 59, 54, 54, 54, 57, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153414431Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"[2]R generation[0..12]: [30, 24, 30, 29, 28, 30, 30, 28, 28, 28, 30, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.15342766Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153441429Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153454547Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs",""Wait for a query_work_queue command to finish before sending IO :"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153475285Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Next flush: 61","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153487944Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153503513Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"":"downstairs","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153516641Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.15352849Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"0","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153541699Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"1","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153554678Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"client":"2","":"downstairs","session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153567627Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153579625Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"6cd151f6-a3e5-4cbb-866e-bcfb1017b6cf is now active with session: 175d52ae-0e6b-495f-bb06-d78a94a4bc60","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153592104Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"Set Active after no reconciliation"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153607983Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315,"session_id":"175d52ae-0e6b-495f-bb06-d78a94a4bc60"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:21.153620041Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 52 len 12288 data: 4 4 4 02/30 Read block 211 len 40960 03/30 Write block 249 len 16384 data: 5 5 6 7 04/30 Write block 333 len 32768 data: 3 3 2 2 4 4 4 5 05/30 Write block 22 len 36864 data: 4 4 4 5 5 4 4 4 4 06/30 Write block 430 len 32768 data: 3 3 3 4 5 4 5 5 07/30 Write block 434 len 24576 data: 6 5 6 6 5 4 08/30 Write block 296 len 32768 data: 3 3 3 3 4 4 4 3 09/30 Read block 312 len 16384 10/30 Read block 446 len 32768 11/30 Flush 12/30 Write block 115 len 12288 data: 3 3 4 13/30 Read block 267 len 28672 14/30 Write block 248 len 4096 data: 6 15/30 Write block 86 len 20480 data: 3 3 2 3 4 16/30 Read block 496 len 20480 17/30 Write block 207 len 36864 data: 3 4 3 3 3 3 2 2 2 18/30 Read block 219 len 24576 19/30 Write block 516 len 24576 data: 5 4 4 4 5 4 20/30 Write block 199 len 24576 data: 2 2 2 2 3 4 21/30 Write block 485 len 20480 data: 5 4 4 4 4 22/30 Write block 306 len 20480 data: 4 4 5 6 5 23/30 Flush 24/30 Write block 87 len 40960 data: 4 3 4 5 4 3 4 3 4 4 25/30 Read block 461 len 24576 26/30 Read block 376 len 32768 27/30 Read block 355 len 40960 28/30 Read block 130 len 36864 29/30 Write block 381 len 8192 data: 5 4 30/30 Write block 243 len 32768 data: 5 5 5 5 5 7 6 6 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:21.17335401Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1315} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 58 58 58 F F F 1 020-039 32 32 32 61 61 61 F F F 2 040-059 32 32 32 61 61 61 F F F 3 060-079 29 29 29 56 56 56 F F F 4 080-099 32 32 32 62 62 62 T T T 5 100-119 32 32 32 62 62 62 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 54 54 54 F F F 8 160-179 28 28 28 54 54 54 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 32 32 32 62 62 62 F F F 11 220-239 23 23 23 47 47 47 F F F 12 240-259 32 32 32 62 62 62 T T T 13 260-279 30 30 30 58 58 58 F F F 14 280-299 32 32 32 61 61 61 F F F 15 300-319 32 32 32 62 62 62 F F F 16 320-339 32 32 32 61 61 61 F F F 17 340-359 32 32 32 61 61 61 F F F 18 360-379 27 27 27 52 52 52 F F F 19 380-399 31 31 31 60 60 60 T T T 20 400-419 30 30 30 58 58 58 F F F 21 420-439 32 32 32 61 61 61 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 25 25 25 49 49 49 F F F 24 480-499 32 32 32 62 62 62 F F F 25 500-519 32 32 32 62 62 62 F F F 26 520-539 32 32 32 62 62 62 F F F 27 540-559 28 28 28 54 54 54 F F F 28 560-579 29 29 29 56 56 56 F F F 29 580-599 31 31 31 60 60 60 F F F Max gen: 32, Max flush: 62 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 58 58 58 F F F 1 020-039 32 32 32 61 61 61 F F F 2 040-059 32 32 32 61 61 61 F F F 3 060-079 29 29 29 56 56 56 F F F 4 080-099 33 33 33 63 63 63 F F F 5 100-119 32 32 32 62 62 62 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 54 54 54 F F F 8 160-179 28 28 28 54 54 54 F F F 9 180-199 32 32 32 62 62 62 F F F 10 200-219 32 32 32 62 62 62 F F F 11 220-239 23 23 23 47 47 47 F F F 12 240-259 33 33 33 63 63 63 F F F 13 260-279 30 30 30 58 58 58 F F F 14 280-299 32 32 32 61 61 61 F F F 15 300-319 32 32 32 62 62 62 F F F 16 320-339 32 32 32 61 61 61 F F F 17 340-359 32 32 32 61 61 61 F F F 18 360-379 27 27 27 52 52 52 F F F 19 380-399 33 33 33 63 63 63 F F F 20 400-419 30 30 30 58 58 58 F F F 21 420-439 32 32 32 61 61 61 F F F 22 440-459 20 20 20 44 44 44 F F F 23 460-479 25 25 25 49 49 49 F F F 24 480-499 32 32 32 62 62 62 F F F 25 500-519 32 32 32 62 62 62 F F F 26 520-539 32 32 32 62 62 62 F F F 27 540-559 28 28 28 54 54 54 F F F 28 560-579 29 29 29 56 56 56 F F F 29 580-599 31 31 31 60 60 60 F F F Max gen: 33, Max flush: 63 Begin loop 17 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.396670833Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:25.438104719Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.438490871Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.46941384Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.470090064Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.4701327Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.470149669Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.470167087Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323} {"msg":"Crucible ad8dd302-2994-49c4-9dd8-127f06badc69 has session id: 400a1a0e-0a32-4206-9269-f88e43d3bca3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.470783907Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Upstairs opts: Upstairs UUID: ad8dd302-2994-49c4-9dd8-127f06badc69, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.470830642Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Crucible stats registered with UUID: ad8dd302-2994-49c4-9dd8-127f06badc69","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.47084903Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:25.470861929Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471238232Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.47126496Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323} {"msg":"ad8dd302-2994-49c4-9dd8-127f06badc69 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471322004Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471339942Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"0","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471362Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"0","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471375869Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"1","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471390048Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"1","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471403566Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"2","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471421705Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"2","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471446562Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.47146688Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471485568Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45282)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471511776Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"io task","client":"0","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"ds_connection connected from Ok(127.0.0.1:62713)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471538863Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"io task","client":"2","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"ds_connection connected from Ok(127.0.0.1:46379)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.471733104Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"io task","client":"1","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.472297659Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"1","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.472336405Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"0","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.472364523Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"2","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.47290595Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"2","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473037987Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"1","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473084463Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"0","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473437208Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473541198Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"[0]R flush_numbers[0..12]: [58, 61, 61, 56, 63, 62, 59, 54, 54, 62, 62, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473661536Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"[0]R generation[0..12]: [30, 32, 32, 29, 33, 32, 30, 28, 28, 32, 32, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473682154Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473697153Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"[1]R flush_numbers[0..12]: [58, 61, 61, 56, 63, 62, 59, 54, 54, 62, 62, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473710922Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"[1]R generation[0..12]: [30, 32, 32, 29, 33, 32, 30, 28, 28, 32, 32, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.47372439Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473738319Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"[2]R flush_numbers[0..12]: [58, 61, 61, 56, 63, 62, 59, 54, 54, 62, 62, 47]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473756187Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"[2]R generation[0..12]: [30, 32, 32, 29, 33, 32, 30, 28, 28, 32, 32, 23]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473770166Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473783755Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473807142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.47383173Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473854868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473877006Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"":"downstairs","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473899473Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473919601Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"0","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"timeDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"2025-01-28T17:10:25.473942129Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"1","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473968097Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"client":"2","":"downstairs","session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.473990915Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.474004333Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"ad8dd302-2994-49c4-9dd8-127f06badc69 is now active with session: 400a1a0e-0a32-4206-9269-f88e43d3bca3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.474016822Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.474028601Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323,"session_id":"400a1a0e-0a32-4206-9269-f88e43d3bca3"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:25.474043299Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 498 len 36864 data: 4 4 4 4 3 4 6 6 5 02/30 Read block 56 len 8192 03/30 Write block 190 len 36864 data: 4 4 4 4 3 3 3 3 2 04/30 Flush 05/30 Write block 519 len 8192 data: 5 6 06/30 Read block 581 len 36864 07/30 Flush 08/30 Write block 447 len 8192 data: 3 3 09/30 Flush 10/30 Write block 269 len 12288 data: 6 5 4 11/30 Read block 499 len 16384 12/30 Read block 444 len 16384 13/30 Flush 14/30 Write block 164 len 4096 data: 7 15/30 Read block 429 len 20480 16/30 Read block 317 len 12288 17/30 Write block 347 len 8192 data: 6 6 18/30 Write block 512 len 28672 data: 5 6 5 5 6 5 5 19/30 Read block 179 len 4096 20/30 Read block 226 len 8192 21/30 Read block 185 len 16384 22/30 Write block 216 len 8192 data: 2 2 23/30 Write block 438 len 40960 data: 6 5 4 3 4 4 3 4 3 4 24/30 Write block 168 len 28672 data: 6 5 4 5 6 6 6 25/30 Write block 274 len 20480 data: 3 4 4 4 5 26/30 Write block 408 len 24576 data: 3 3 3 3 4 4 27/30 Write block 292 len 40960 data: 3 3 3 2 4 4 4 4 5 5 28/30 Write block 185 len 20480 data: 5 4 4 3 4 29/30 Read block 349 len 40960 30/30 Write block 221 len 24576 data: 3 3 3 3 4 5 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 1 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:25.49159508Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1323} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 58 58 58 F F F 1 020-039 32 32 32 61 61 61 F F F 2 040-059 32 32 32 61 61 61 F F F 3 060-079 29 29 29 56 56 56 F F F 4 080-099 33 33 33 63 63 63 F F F 5 100-119 32 32 32 62 62 62 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 54 54 54 F F F 8 160-179 28 28 28 54 54 54 T T T 9 180-199 34 34 34 64 64 64 T T T 10 200-219 32 32 32 62 62 62 T T T 11 220-239 23 23 23 47 47 47 T T T 12 240-259 33 33 33 63 63 63 F F F 13 260-279 34 34 34 67 67 67 T T T 14 280-299 32 32 32 61 61 61 T T T 15 300-319 32 32 32 62 62 62 T T T 16 320-339 32 32 32 61 61 61 F F F 17 340-359 32 32 32 61 61 61 T T T 18 360-379 27 27 27 52 52 52 F F F 19 380-399 33 33 33 63 63 63 F F F 20 400-419 30 30 30 58 58 58 T T T 21 420-439 32 32 32 61 61 61 T T T 22 440-459 34 34 34 66 66 66 T T T 23 460-479 25 25 25 49 49 49 F F F 24 480-499 34 34 34 64 64 64 F F F 25 500-519 34 34 34 65 65 65 T T T 26 520-539 34 34 34 65 65 65 F F F 27 540-559 28 28 28 54 54 54 F F F 28 560-579 29 29 29 56 56 56 F F F 29 580-599 31 31 31 60 60 60 F F F Max gen: 34, Max flush: 67 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 58 58 58 F F F 1 020-039 32 32 32 61 61 61 F F F 2 040-059 32 32 32 61 61 61 F F F 3 060-079 29 29 29 56 56 56 F F F 4 080-099 33 33 33 63 63 63 F F F 5 100-119 32 32 32 62 62 62 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 54 54 54 F F F 8 160-179 35 35 35 68 68 68 F F F 9 180-199 35 35 35 68 68 68 F F F 10 200-219 35 35 35 68 68 68 F F F 11 220-239 35 35 35 68 68 68 F F F 12 240-259 33 33 33 63 63 63 F F F 13 260-279 35 35 35 68 68 68 F F F 14 280-299 35 35 35 68 68 68 F F F 15 300-319 35 35 35 68 68 68 F F F 16 320-339 32 32 32 61 61 61 F F F 17 340-359 35 35 35 68 68 68 F F F 18 360-379 27 27 27 52 52 52 F F F 19 380-399 33 33 33 63 63 63 F F F 20 400-419 35 35 35 68 68 68 F F F 21 420-439 35 35 35 68 68 68 F F F 22 440-459 35 35 35 68 68 68 F F F 23 460-479 25 25 25 49 49 49 F F F 24 480-499 34 34 34 64 64 64 F F F 25 500-519 35 35 35 68 68 68 F F F 26 520-539 34 34 34 65 65 65 F F F 27 540-559 28 28 28 54 54 54 F F F 28 560-579 29 29 29 56 56 56 F F F 29 580-599 31 31 31 60 60 60 F F F Max gen: 35, Max flush: 68 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.789570725Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.834032746Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.834660785Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.834700551Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.834720839Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.834735858Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331} {"msg":"Crucible c344a8bc-870d-4383-a222-2f7db2235a10 has session id: 31b86b28-818c-4352-82df-fe4f261e43b4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.835264706Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Upstairs opts: Upstairs UUID: c344a8bc-870d-4383-a222-2f7db2235a10, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.835310922Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Crucible stats registered with UUID: c344a8bc-870d-4383-a222-2f7db2235a10","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.835334379Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:29.835354507Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.835791185Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.835816792Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331} {"msg":"c344a8bc-870d-4383-a222-2f7db2235a10 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.835873677Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.835900914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"0","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.835924882Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"0","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.83594073Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"1","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.835957799Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"1","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.835970907Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"2","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.835993415Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"2","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.836024912Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.836062568Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.836105954Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35867)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.836132331Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"io task","client":"1","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:60116)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.836159039Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"io task","client":"2","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:34740)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.836187016Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"io task","client":"0","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.836885838Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"1","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.836928164Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"0","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.8373816Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"2","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.837595919Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"1","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.837624756Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"0","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.83799303Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"2","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838187422Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.83820822Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"[0]R flush_numbers[0..12]: [58, 61, 61, 56, 63, 62, 59, 54, 68, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838526788Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"[0]R generation[0..12]: [30, 32, 32, 29, 33, 32, 30, 28, 35, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838549706Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838564185Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"[1]R flush_numbers[0..12]: [58, 61, 61, 56, 63, 62, 59, 54, 68, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838578004Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"[1]R generation[0..12]: [30, 32, 32, 29, 33, 32, 30, 28, 35, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838591452Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838605471Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"[2]R flush_numbers[0..12]: [58, 61, 61, 56, 63, 62, 59, 54, 68, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838619199Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"[2]R generation[0..12]: [30, 32, 32, 29, 33, 32, 30, 28, 35, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838632508Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838646227Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838665045Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838677894Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838690482Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838703061Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"":"downstairs","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.83871621Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838727799Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"0","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838744117Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"1","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838757326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"client":"2","":"downstairs","session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838770495Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-28T17:10:29.838782513Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"c344a8bc-870d-4383-a222-2f7db2235a10 is now active with session: 31b86b28-818c-4352-82df-fe4f261e43b4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838798152Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.838809801Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331,"session_id":"31b86b28-818c-4352-82df-fe4f261e43b4"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:29.83882167Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 279 len 4096 data: 5 02/30 Flush 03/30 Write block 391 len 24576 data: 3 4 4 2 2 2 04/30 Read block 31 len 16384 05/30 Read block 447 len 40960 06/30 Write block 334 len 12288 data: 4 3 3 07/30 Write block 59 len 4096 data: 3 08/30 Write block 422 len 40960 data: 3 3 3 3 3 3 2 3 4 4 09/30 Read block 327 len 28672 10/30 Read block 77 len 36864 11/30 Read block 548 len 12288 12/30 Write block 546 len 40960 data: 2 2 2 2 4 4 4 4 4 3 13/30 Write block 310 len 32768 data: 6 6 5 4 5 6 5 5 14/30 Write block 358 len 16384 data: 5 4 4 3 15/30 Write block 511 len 16384 data: 5 6 7 6 16/30 Read block 200 len 40960 17/30 Write block 175 len 8192 data: 6 5 18/30 Flush 19/30 Flush 20/30 Write block 233 len 24576 data: 3 3 4 3 3 4 21/30 Write block 358 len 40960 data: 6 5 5 4 4 5 6 6 4 4 22/30 Read block 524 len 32768 23/30 Write block 503 len 20480 data: 5 7 7 6 6 24/30 Read block 159 len 8192 25/30 Read block 462 len 28672 26/30 Write block 175 len 4096 data: 7 27/30 Read block 67 len 24576 28/30 Write block 196 len 4096 data: 4 29/30 Read block 237 len 28672 30/30 Write block 34 len 32768 data: 4 3 3 4 4 4 4 4 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 6 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:29.858115491Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1331} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 58 58 58 F F F 1 020-039 32 32 32 61 61 61 T T T 2 040-059 36 36 36 70 70 70 T T T 3 060-079 29 29 29 56 56 56 F F F 4 080-099 33 33 33 63 63 63 F F F 5 100-119 32 32 32 62 62 62 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 54 54 54 F F F 8 160-179 36 36 36 70 70 70 T T T 9 180-199 35 35 35 68 68 68 T T T 10 200-219 35 35 35 68 68 68 F F F 11 220-239 35 35 35 68 68 68 T T T 12 240-259 33 33 33 63 63 63 F F F 13 260-279 36 36 36 69 69 69 F F F 14 280-299 35 35 35 68 68 68 F F F 15 300-319 36 36 36 70 70 70 F F F 16 320-339 36 36 36 70 70 70 F F F 17 340-359 36 36 36 70 70 70 T T T 18 360-379 36 36 36 70 70 70 T T T 19 380-399 36 36 36 70 70 70 F F F 20 400-419 35 35 35 68 68 68 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 35 35 35 68 68 68 F F F 23 460-479 25 25 25 49 49 49 F F F 24 480-499 34 34 34 64 64 64 F F F 25 500-519 36 36 36 70 70 70 T T T 26 520-539 34 34 34 65 65 65 F F F 27 540-559 36 36 36 70 70 70 F F F 28 560-579 29 29 29 56 56 56 F F F 29 580-599 31 31 31 60 60 60 F F F Max gen: 36, Max flush: 70 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 58 58 58 F F F 1 020-039 37 37 37 71 71 71 F F F 2 040-059 37 37 37 71 71 71 F F F 3 060-079 29 29 29 56 56 56 F F F 4 080-099 33 33 33 63 63 63 F F F 5 100-119 32 32 32 62 62 62 F F F 6 120-139 30 30 30 59 59 59 F F F 7 140-159 28 28 28 54 54 54 F F F 8 160-179 37 37 37 71 71 71 F F F 9 180-199 37 37 37 71 71 71 F F F 10 200-219 35 35 35 68 68 68 F F F 11 220-239 37 37 37 71 71 71 F F F 12 240-259 33 33 33 63 63 63 F F F 13 260-279 36 36 36 69 69 69 F F F 14 280-299 35 35 35 68 68 68 F F F 15 300-319 36 36 36 70 70 70 F F F 16 320-339 36 36 36 70 70 70 F F F 17 340-359 37 37 37 71 71 71 F F F 18 360-379 37 37 37 71 71 71 F F F 19 380-399 36 36 36 70 70 70 F F F 20 400-419 35 35 35 68 68 68 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 35 35 35 68 68 68 F F F 23 460-479 25 25 25 49 49 49 F F F 24 480-499 34 34 34 64 64 64 F F F 25 500-519 37 37 37 71 71 71 F F F 26 520-539 34 34 34 65 65 65 F F F 27 540-559 36 36 36 70 70 70 F F F 28 560-579 29 29 29 56 56 56 F F F 29 580-599 31 31 31 60 60 60 F F F Max gen: 37, Max flush: 71 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.10891188Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.15204167Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.152650971Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.152690707Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.152705966Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.152720154Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339} {"msg":"Crucible 310b22cf-8b2d-401a-ab25-51bd489e0a3b has session id: 2a6abf82-fa7e-43c5-b54d-d719ddf84a7f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.153244363Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Upstairs opts: Upstairs UUID: 310b22cf-8b2d-401a-ab25-51bd489e0a3b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.15327754Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Crucible stats registered with UUID: 310b22cf-8b2d-401a-ab25-51bd489e0a3b","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.153291998Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:34.153304567Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.153810628Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.153843115Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339} {"msg":"310b22cf-8b2d-401a-ab25-51bd489e0a3b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.153883521Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.153904149Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"0","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.153943815Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"0","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.153973732Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"1","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.15399968Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"1","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.154022347Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"2","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.154046015Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"2","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.154068013Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.154091511Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.154114348Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44844)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.154139466Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"io task","client":"0","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45911)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.154164624Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"io task","client":"2","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63687)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.154190231Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"io task","client":"1","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.15491346Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"0","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.154945208Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"2","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.154972465Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"1","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.155496324Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"0","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.155605833Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"1","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.155646119Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"2","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156101275Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156220103Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"[0]R flush_numbers[0..12]: [58, 71, 71, 56, 63, 62, 59, 54, 71, 71, 68, 71]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156284407Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"[0]R generation[0..12]: [30, 37, 37, 29, 33, 32, 30, 28, 37, 37, 35, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156305235Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156320134Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"[1]R flush_numbers[0..12]: [58, 71, 71, 56, 63, 62, 59, 54, 71, 71, 68, 71]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156333592Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"[1]R generation[0..12]: [30, 37, 37, 29, 33, 32, 30, 28, 37, 37, 35, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156346961Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.15636093Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"[2]R flush_numbers[0..12]: [58, 71, 71, 56, 63, 62, 59, 54, 71, 71, 68, 71]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156374328Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"[2]R generation[0..12]: [30, 37, 37, 29, 33, 32, 30, 28, 37, 37, 35, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156387557Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156401306Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156418754Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:10:34.156432033Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156447221Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.1564598Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"":"downstairs","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156472829Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156484678Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"0","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156497906Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"1","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156517684Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"client":"2","":"downstairs","session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156532973Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156545142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"310b22cf-8b2d-401a-ab25-51bd489e0a3b is now active with session: 2a6abf82-fa7e-43c5-b54d-d719ddf84a7f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156557831Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156569699Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339,"session_id":"2a6abf82-fa7e-43c5-b54d-d719ddf84a7f"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:34.156581578Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 228 len 16384 data: 5 4 3 2 02/30 Read block 316 len 36864 03/30 Flush 04/30 Write block 310 len 36864 data: 7 7 6 5 6 7 6 6 5 05/30 Write block 15 len 12288 data: 4 5 3 06/30 Read block 381 len 32768 07/30 Write block 137 len 20480 data: 7 7 7 7 6 08/30 Write block 204 len 16384 data: 5 4 4 4 09/30 Read block 479 len 28672 10/30 Read block 170 len 32768 11/30 Read block 447 len 8192 12/30 Read block 518 len 28672 13/30 Write block 554 len 16384 data: 5 4 3 3 14/30 Read block 316 len 4096 15/30 Read block 551 len 32768 16/30 Read block 379 len 36864 17/30 Read block 359 len 32768 18/30 Read block 213 len 12288 19/30 Read block 104 len 4096 20/30 Write block 26 len 32768 data: 6 5 5 5 5 4 3 3 21/30 Read block 278 len 8192 22/30 Read block 212 len 40960 23/30 Read block 358 len 36864 24/30 Read block 428 len 32768 25/30 Read block 472 len 24576 26/30 Write block 475 len 32768 data: 2 3 5 6 6 6 5 7 27/30 Write block 104 len 4096 data: 3 28/30 Read block 202 len 28672 29/30 Read block 48 len 20480 30/30 Read block 139 len 12288 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 9 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 27 26 26 79 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:34.176410687Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1339} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 58 58 58 T T T 1 020-039 37 37 37 71 71 71 T T T 2 040-059 37 37 37 71 71 71 F F F 3 060-079 29 29 29 56 56 56 F F F 4 080-099 33 33 33 63 63 63 F F F 5 100-119 32 32 32 62 62 62 T T T 6 120-139 30 30 30 59 59 59 T T T 7 140-159 28 28 28 54 54 54 T T T 8 160-179 37 37 37 71 71 71 F F F 9 180-199 37 37 37 71 71 71 F F F 10 200-219 35 35 35 68 68 68 T T T 11 220-239 38 38 38 72 72 72 F F F 12 240-259 33 33 33 63 63 63 F F F 13 260-279 36 36 36 69 69 69 F F F 14 280-299 35 35 35 68 68 68 F F F 15 300-319 36 36 36 70 70 70 T T T 16 320-339 36 36 36 70 70 70 F F F 17 340-359 37 37 37 71 71 71 F F F 18 360-379 37 37 37 71 71 71 F F F 19 380-399 36 36 36 70 70 70 F F F 20 400-419 35 35 35 68 68 68 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 35 35 35 68 68 68 F F F 23 460-479 25 25 25 49 49 49 T T T 24 480-499 34 34 34 64 64 64 T T T 25 500-519 37 37 37 71 71 71 F F F 26 520-539 34 34 34 65 65 65 F F F 27 540-559 36 36 36 70 70 70 T T T 28 560-579 29 29 29 56 56 56 F F F 29 580-599 31 31 31 60 60 60 F F F Max gen: 38, Max flush: 72 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 39 39 39 73 73 73 F F F 1 020-039 39 39 39 73 73 73 F F F 2 040-059 37 37 37 71 71 71 F F F 3 060-079 29 29 29 56 56 56 F F F 4 080-099 33 33 33 63 63 63 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 39 39 39 73 73 73 F F F 7 140-159 39 39 39 73 73 73 F F F 8 160-179 37 37 37 71 71 71 F F F 9 180-199 37 37 37 71 71 71 F F F 10 200-219 39 39 39 73 73 73 F F F 11 220-239 38 38 38 72 72 72 F F F 12 240-259 33 33 33 63 63 63 F F F 13 260-279 36 36 36 69 69 69 F F F 14 280-299 35 35 35 68 68 68 F F F 15 300-319 39 39 39 73 73 73 F F F 16 320-339 36 36 36 70 70 70 F F F 17 340-359 37 37 37 71 71 71 F F F 18 360-379 37 37 37 71 71 71 F F F 19 380-399 36 36 36 70 70 70 F F F 20 400-419 35 35 35 68 68 68 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 35 35 35 68 68 68 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 39 39 39 73 73 73 F F F 25 500-519 37 37 37 71 71 71 F F F 26 520-539 34 34 34 65 65 65 F F F 27 540-559 39 39 39 73 73 73 F F F 28 560-579 29 29 29 56 56 56 F F F 29 580-599 31 31 31 60 60 60 F F F Max gen: 39, Max flush: 73 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.440313198Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.4835361Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.484115393Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.4841548Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.484169378Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.484183717Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347} {"msg":"Crucible 7e338518-a0c5-40fe-8975-905d879236ee has session id: 10a66f5a-5cb1-4a59-92d3-99164aca8a45","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.484708206Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Upstairs opts: Upstairs UUID: 7e338518-a0c5-40fe-8975-905d879236ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.484754251Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Crucible stats registered with UUID: 7e338518-a0c5-40fe-8975-905d879236ee","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.484777189Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:38.484797937Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485271671Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485307037Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347} {"msg":"7e338518-a0c5-40fe-8975-905d879236ee active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485331455Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485353353Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"0","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.48538394Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"0","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485411747Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"1","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485440254Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"1","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485466962Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"2","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485494339Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"2","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485517517Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485543314Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485572222Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"ds_connection connected from Ok(127.0.0.1:45025)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485614697Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"io task","client":"1","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"ds_connection connected from Ok(127.0.0.1:54475)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485640955Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"io task","client":"2","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"ds_connection connected from Ok(127.0.0.1:57754)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.485714008Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"io task","client":"0","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.486285722Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"0","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.486315489Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"2","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.486973945Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"0","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.487016011Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"1","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.487032929Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"2","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.487572067Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.487682016Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"1","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.487706814Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"[0]R flush_numbers[0..12]: [73, 73, 71, 56, 63, 73, 73, 73, 71, 71, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488203765Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"[0]R generation[0..12]: [39, 39, 37, 29, 33, 39, 39, 39, 37, 37, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488228433Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488247151Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"[1]R flush_numbers[0..12]: [73, 73, 71, 56, 63, 73, 73, 73, 71, 71, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.48826103Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"[1]R generation[0..12]: [39, 39, 37, 29, 33, 39, 39, 39, 37, 37, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488274858Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488288737Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"[2]R flush_numbers[0..12]: [73, 73, 71, 56, 63, 73, 73, 73, 71, 71, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488302096Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"[2]R generation[0..12]: [39, 39, 37, 29, 33, 39, 39, 39, 37, 37, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488315224Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488328633Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Max found gen is 40","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2025-01-28T17:10:38.488347431Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.48836052Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488373139Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488385577Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"":"downstairs","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488398246Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488409925Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"0","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488422824Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"1","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488435543Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"client":"2","":"downstairs","session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488450391Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.48846251Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"7e338518-a0c5-40fe-8975-905d879236ee is now active with session: 10a66f5a-5cb1-4a59-92d3-99164aca8a45","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488474929Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-01-28T17:10:38.488486798Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347,"session_id":"10a66f5a-5cb1-4a59-92d3-99164aca8a45"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:38.488501916Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 140 len 16384 data: 8 7 4 4 02/30 Read block 239 len 4096 03/30 Read block 14 len 16384 04/30 Write block 580 len 16384 data: 5 5 6 6 05/30 Write block 158 len 12288 data: 6 6 6 06/30 Read block 571 len 24576 07/30 Write block 56 len 32768 data: 4 3 3 4 4 5 5 5 08/30 Read block 187 len 4096 09/30 Flush 10/30 Write block 529 len 36864 data: 5 6 6 5 5 2 3 4 5 11/30 Read block 117 len 16384 12/30 Write block 25 len 32768 data: 6 7 6 6 6 6 5 4 13/30 Flush 14/30 Write block 588 len 28672 data: 3 4 2 2 2 2 2 15/30 Read block 407 len 36864 16/30 Write block 530 len 16384 data: 7 7 6 6 17/30 Write block 382 len 20480 data: 5 5 5 4 4 18/30 Read block 422 len 20480 19/30 Read block 271 len 16384 20/30 Read block 456 len 40960 21/30 Write block 385 len 36864 data: 5 5 4 4 3 3 4 5 5 22/30 Read block 540 len 20480 23/30 Write block 320 len 12288 data: 2 3 5 24/30 Read block 330 len 28672 25/30 Read block 295 len 32768 26/30 Write block 77 len 28672 data: 3 3 4 3 4 4 4 27/30 Read block 64 len 28672 28/30 Read block 187 len 20480 29/30 Write block 0 len 24576 data: 2 2 2 2 3 3 30/30 Read block 243 len 24576 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 7 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 17 15 15 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:38.509543467Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1347} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 39 39 39 73 73 73 T T T 1 020-039 40 40 40 75 75 75 F F F 2 040-059 40 40 40 74 74 74 F F F 3 060-079 40 40 40 74 74 74 T T T 4 080-099 33 33 33 63 63 63 T T T 5 100-119 39 39 39 73 73 73 F F F 6 120-139 39 39 39 73 73 73 F F F 7 140-159 40 40 40 74 74 74 F F F 8 160-179 40 40 40 74 74 74 F F F 9 180-199 37 37 37 71 71 71 F F F 10 200-219 39 39 39 73 73 73 F F F 11 220-239 38 38 38 72 72 72 F F F 12 240-259 33 33 33 63 63 63 F F F 13 260-279 36 36 36 69 69 69 F F F 14 280-299 35 35 35 68 68 68 F F F 15 300-319 39 39 39 73 73 73 F F F 16 320-339 36 36 36 70 70 70 T T T 17 340-359 37 37 37 71 71 71 F F F 18 360-379 37 37 37 71 71 71 F F F 19 380-399 36 36 36 70 70 70 T T T 20 400-419 35 35 35 68 68 68 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 35 35 35 68 68 68 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 39 39 39 73 73 73 F F F 25 500-519 37 37 37 71 71 71 F F F 26 520-539 40 40 40 75 75 75 T T T 27 540-559 39 39 39 73 73 73 F F F 28 560-579 29 29 29 56 56 56 F F F 29 580-599 40 40 40 74 74 74 T T T Max gen: 40, Max flush: 75 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 76 76 76 F F F 1 020-039 40 40 40 75 75 75 F F F 2 040-059 40 40 40 74 74 74 F F F 3 060-079 41 41 41 76 76 76 F F F 4 080-099 41 41 41 76 76 76 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 39 39 39 73 73 73 F F F 7 140-159 40 40 40 74 74 74 F F F 8 160-179 40 40 40 74 74 74 F F F 9 180-199 37 37 37 71 71 71 F F F 10 200-219 39 39 39 73 73 73 F F F 11 220-239 38 38 38 72 72 72 F F F 12 240-259 33 33 33 63 63 63 F F F 13 260-279 36 36 36 69 69 69 F F F 14 280-299 35 35 35 68 68 68 F F F 15 300-319 39 39 39 73 73 73 F F F 16 320-339 41 41 41 76 76 76 F F F 17 340-359 37 37 37 71 71 71 F F F 18 360-379 37 37 37 71 71 71 F F F 19 380-399 41 41 41 76 76 76 F F F 20 400-419 35 35 35 68 68 68 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 35 35 35 68 68 68 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 39 39 39 73 73 73 F F F 25 500-519 37 37 37 71 71 71 F F F 26 520-539 41 41 41 76 76 76 F F F 27 540-559 39 39 39 73 73 73 F F F 28 560-579 29 29 29 56 56 56 F F F 29 580-599 41 41 41 76 76 76 F F F Max gen: 41, Max flush: 76 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.752593719Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.796067366Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.796718413Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.796758469Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.796773757Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.796788266Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355} {"msg":"Crucible 2b572204-eab4-4df8-9fe1-bff9e59b77e3 has session id: a89276ae-142f-45fa-a48e-acf92b22fd89","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.797290737Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Upstairs opts: Upstairs UUID: 2b572204-eab4-4df8-9fe1-bff9e59b77e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.797338342Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Crucible stats registered with UUID: 2b572204-eab4-4df8-9fe1-bff9e59b77e3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.79736224Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:42.797381648Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.797903697Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.797931434Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355} {"msg":"2b572204-eab4-4df8-9fe1-bff9e59b77e3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798012567Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798032765Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"0","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798055652Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"0","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.79807417Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"1","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798088639Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"1","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798101678Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"2","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798115546Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"2","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798137124Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798160942Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.79818393Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"ds_connection connected from Ok(127.0.0.1:51966)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798216787Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"io task","client":"1","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"ds_connection connected from Ok(127.0.0.1:52999)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798244194Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"io task","client":"2","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"ds_connection connected from Ok(127.0.0.1:52465)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798409018Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"io task","client":"0","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798965594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"0","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.798993771Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"1","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.79900885Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"2","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.799579944Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"0","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.799662726Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"1","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.799687593Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"2","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800183115Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800287815Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"[0]R flush_numbers[0..12]: [76, 75, 74, 76, 76, 73, 73, 74, 74, 71, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800311263Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"[0]R generation[0..12]: [41, 40, 40, 41, 41, 39, 39, 40, 40, 37, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800325851Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.80034018Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"[1]R flush_numbers[0..12]: [76, 75, 74, 76, 76, 73, 73, 74, 74, 71, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800353979Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"[1]R generation[0..12]: [41, 40, 40, 41, 41, 39, 39, 40, 40, 37, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800367147Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800380666Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"[2]R flush_numbers[0..12]: [76, 75, 74, 76, 76, 73, 73, 74, 74, 71, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800393755Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"[2]R generation[0..12]: [41, 40, 40, 41, 41, 39, 39, 40, 40, 37, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800406813Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800420312Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800432901Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800450189Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800463478Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800476207Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"":"downstairs","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800489055Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800512263Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"0","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800525312Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"1","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800537921Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"client":"2","":"downstairs","session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800550659Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800562578Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"2b572204-eab4-4df8-9fe1-bff9e59b77e3 is now active with session: a89276ae-142f-45fa-a48e-acf92b22fd89","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800575097Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800586776Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355,"session_id":"a89276ae-142f-45fa-a48e-acf92b22fd89"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:42.800598505Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 43 len 8192 data: 4 4 02/30 Write block 342 len 20480 data: 6 6 6 6 5 03/30 Flush 04/30 Write block 137 len 16384 data: 8 8 8 9 05/30 Read block 451 len 8192 06/30 Read block 499 len 28672 07/30 Write block 568 len 16384 data: 3 3 4 4 08/30 Write block 513 len 20480 data: 8 7 6 7 6 09/30 Write block 57 len 28672 data: 4 4 5 5 6 6 6 10/30 Read block 269 len 40960 11/30 Read block 119 len 28672 12/30 Read block 148 len 8192 13/30 Flush 14/30 Write block 366 len 24576 data: 5 5 4 3 3 3 15/30 Read block 80 len 28672 16/30 Read block 19 len 16384 17/30 Write block 10 len 28672 data: 5 5 5 5 4 5 6 18/30 Write block 533 len 32768 data: 7 3 4 5 6 5 5 4 19/30 Write block 250 len 12288 data: 7 7 8 20/30 Read block 418 len 32768 21/30 Read block 353 len 28672 22/30 Read block 24 len 28672 23/30 Write block 397 len 28672 data: 2 3 3 5 5 5 7 24/30 Read block 184 len 40960 25/30 Write block 47 len 24576 data: 3 2 2 2 3 5 26/30 Write block 140 len 28672 data: 10 8 5 5 3 3 3 27/30 Write block 378 len 28672 data: 3 4 4 6 6 6 6 28/30 Write block 307 len 40960 data: 5 6 7 8 8 7 6 7 8 7 29/30 Read block 226 len 36864 30/30 Write block 285 len 40960 data: 2 2 2 2 3 3 3 4 4 4 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 6 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:42.820171369Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1355} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 76 76 76 T T T 1 020-039 40 40 40 75 75 75 F F F 2 040-059 42 42 42 78 78 78 T T T 3 060-079 42 42 42 78 78 78 F F F 4 080-099 41 41 41 76 76 76 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 42 42 42 78 78 78 F F F 7 140-159 42 42 42 78 78 78 T T T 8 160-179 40 40 40 74 74 74 F F F 9 180-199 37 37 37 71 71 71 F F F 10 200-219 39 39 39 73 73 73 F F F 11 220-239 38 38 38 72 72 72 F F F 12 240-259 33 33 33 63 63 63 T T T 13 260-279 36 36 36 69 69 69 F F F 14 280-299 35 35 35 68 68 68 T T T 15 300-319 39 39 39 73 73 73 T T T 16 320-339 41 41 41 76 76 76 F F F 17 340-359 42 42 42 77 77 77 F F F 18 360-379 37 37 37 71 71 71 T T T 19 380-399 41 41 41 76 76 76 T T T 20 400-419 35 35 35 68 68 68 T T T 21 420-439 36 36 36 70 70 70 F F F 22 440-459 35 35 35 68 68 68 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 39 39 39 73 73 73 F F F 25 500-519 42 42 42 78 78 78 F F F 26 520-539 41 41 41 76 76 76 T T T 27 540-559 39 39 39 73 73 73 T T T 28 560-579 42 42 42 78 78 78 F F F 29 580-599 41 41 41 76 76 76 F F F Max gen: 42, Max flush: 78 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 79 79 79 F F F 1 020-039 40 40 40 75 75 75 F F F 2 040-059 43 43 43 79 79 79 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 41 41 41 76 76 76 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 42 42 42 78 78 78 F F F 7 140-159 43 43 43 79 79 79 F F F 8 160-179 40 40 40 74 74 74 F F F 9 180-199 37 37 37 71 71 71 F F F 10 200-219 39 39 39 73 73 73 F F F 11 220-239 38 38 38 72 72 72 F F F 12 240-259 43 43 43 79 79 79 F F F 13 260-279 36 36 36 69 69 69 F F F 14 280-299 43 43 43 79 79 79 F F F 15 300-319 43 43 43 79 79 79 F F F 16 320-339 41 41 41 76 76 76 F F F 17 340-359 42 42 42 77 77 77 F F F 18 360-379 43 43 43 79 79 79 F F F 19 380-399 43 43 43 79 79 79 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 35 35 35 68 68 68 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 39 39 39 73 73 73 F F F 25 500-519 42 42 42 78 78 78 F F F 26 520-539 43 43 43 79 79 79 F F F 27 540-559 43 43 43 79 79 79 F F F 28 560-579 42 42 42 78 78 78 F F F 29 580-599 41 41 41 76 76 76 F F F Max gen: 43, Max flush: 79 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.084059983Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.127680835Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.128358659Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.128405335Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.128423313Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.128440951Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363} {"msg":"Crucible e920ff2f-563b-4a13-afb3-b7d0ea376034 has session id: e9f0ad23-4873-4a16-80ca-945ce219b790","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129087358Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Upstairs opts: Upstairs UUID: e920ff2f-563b-4a13-afb3-b7d0ea376034, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129136323Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Crucible stats registered with UUID: e920ff2f-563b-4a13-afb3-b7d0ea376034","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129164041Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:47.129184839Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129618656Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129644384Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363} {"msg":"e920ff2f-563b-4a13-afb3-b7d0ea376034 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129694789Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129720826Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"0","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129743664Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"0","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129757763Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"1","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129772072Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"1","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.12978515Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"2","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129801199Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"2","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129814388Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129828046Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.129841385Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"ds_connection connected from Ok(127.0.0.1:45149)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.130004539Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"io task","client":"1","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"ds_connection connected from Ok(127.0.0.1:37961)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.130050765Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"io task","client":"2","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"ds_connection connected from Ok(127.0.0.1:42978)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.13009253Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"io task","client":"0","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.130794012Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"1","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.130822279Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"0","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.130837518Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"2","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.131507453Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"1","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.13153721Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"2","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.131553618Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"0","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132103434Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132138171Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"[0]R flush_numbers[0..12]: [79, 75, 79, 78, 76, 73, 78, 79, 74, 71, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132162259Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"[0]R generation[0..12]: [43, 40, 43, 42, 41, 39, 42, 43, 40, 37, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132192326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132207564Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"[1]R flush_numbers[0..12]: [79, 75, 79, 78, 76, 73, 78, 79, 74, 71, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132221063Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"[1]R generation[0..12]: [43, 40, 43, 42, 41, 39, 42, 43, 40, 37, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132239021Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.13225301Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"[2]R flush_numbers[0..12]: [79, 75, 79, 78, 76, 73, 78, 79, 74, 71, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132266369Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"[2]R generation[0..12]: [43, 40, 43, 42, 41, 39, 42, 43, 40, 37, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132284087Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132309145Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:10:47.132330692Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.1323589Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132382278Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132407225Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"":"downstairs","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132432513Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.1324565Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"0","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132482118Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"1","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132507855Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"client":"2","":"downstairs","session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132535543Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.13256033Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"e920ff2f-563b-4a13-afb3-b7d0ea376034 is now active with session: e9f0ad23-4873-4a16-80ca-945ce219b790","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132593757Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132618715Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363,"session_id":"e9f0ad23-4873-4a16-80ca-945ce219b790"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:47.132633563Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 378 len 12288 data: 4 5 5 02/30 Write block 258 len 36864 data: 4 3 3 3 4 4 5 6 6 03/30 Write block 131 len 12288 data: 3 3 3 04/30 Read block 33 len 8192 05/30 Read block 124 len 40960 06/30 Read block 291 len 32768 07/30 Read block 19 len 12288 08/30 Write block 585 len 8192 data: 7 4 09/30 Flush 10/30 Write block 558 len 20480 data: 3 3 3 2 3 11/30 Write block 271 len 40960 data: 5 2 2 4 5 5 5 6 6 4 12/30 Read block 111 len 8192 13/30 Write block 329 len 20480 data: 4 3 3 3 4 14/30 Flush 15/30 Write block 387 len 24576 data: 5 5 4 4 5 6 16/30 Read block 186 len 24576 17/30 Flush 18/30 Write block 503 len 16384 data: 6 8 8 7 19/30 Read block 151 len 16384 20/30 Read block 584 len 12288 21/30 Write block 10 len 28672 data: 6 6 6 6 5 6 7 22/30 Read block 545 len 12288 23/30 Flush 24/30 Write block 172 len 36864 data: 7 7 7 8 6 6 5 5 6 25/30 Read block 119 len 40960 26/30 Read block 375 len 24576 27/30 Write block 187 len 32768 data: 5 4 5 5 5 5 5 4 28/30 Read block 250 len 8192 29/30 Write block 203 len 32768 data: 4 6 5 5 5 5 4 4 30/30 Read block 192 len 24576 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 7 6 5 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:47.152852824Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1363} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 83 83 83 F F F 1 020-039 40 40 40 75 75 75 F F F 2 040-059 43 43 43 79 79 79 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 41 41 41 76 76 76 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 44 44 44 80 80 80 F F F 7 140-159 43 43 43 79 79 79 F F F 8 160-179 40 40 40 74 74 74 T T T 9 180-199 37 37 37 71 71 71 T T T 10 200-219 39 39 39 73 73 73 T T T 11 220-239 38 38 38 72 72 72 F F F 12 240-259 44 44 44 80 80 80 F F F 13 260-279 44 44 44 81 81 81 F F F 14 280-299 44 44 44 81 81 81 F F F 15 300-319 43 43 43 79 79 79 F F F 16 320-339 44 44 44 81 81 81 F F F 17 340-359 42 42 42 77 77 77 F F F 18 360-379 44 44 44 80 80 80 F F F 19 380-399 44 44 44 82 82 82 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 35 35 35 68 68 68 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 39 39 39 73 73 73 F F F 25 500-519 44 44 44 83 83 83 F F F 26 520-539 43 43 43 79 79 79 F F F 27 540-559 44 44 44 81 81 81 F F F 28 560-579 44 44 44 81 81 81 F F F 29 580-599 44 44 44 80 80 80 F F F Max gen: 44, Max flush: 83 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 83 83 83 F F F 1 020-039 40 40 40 75 75 75 F F F 2 040-059 43 43 43 79 79 79 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 41 41 41 76 76 76 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 44 44 44 80 80 80 F F F 7 140-159 43 43 43 79 79 79 F F F 8 160-179 45 45 45 84 84 84 F F F 9 180-199 45 45 45 84 84 84 F F F 10 200-219 45 45 45 84 84 84 F F F 11 220-239 38 38 38 72 72 72 F F F 12 240-259 44 44 44 80 80 80 F F F 13 260-279 44 44 44 81 81 81 F F F 14 280-299 44 44 44 81 81 81 F F F 15 300-319 43 43 43 79 79 79 F F F 16 320-339 44 44 44 81 81 81 F F F 17 340-359 42 42 42 77 77 77 F F F 18 360-379 44 44 44 80 80 80 F F F 19 380-399 44 44 44 82 82 82 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 35 35 35 68 68 68 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 39 39 39 73 73 73 F F F 25 500-519 44 44 44 83 83 83 F F F 26 520-539 43 43 43 79 79 79 F F F 27 540-559 44 44 44 81 81 81 F F F 28 560-579 44 44 44 81 81 81 F F F 29 580-599 44 44 44 80 80 80 F F F Max gen: 45, Max flush: 84 Begin loop 23 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.373647523Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:51.415404617Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.41578369Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.446731817Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.447293852Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.447332068Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.447347047Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.447361305Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371} {"msg":"Crucible d8f70a36-7526-4921-ab90-a517a16c671a has session id: 06008ac9-4ab3-4f29-84d7-dd1ac0f59c93","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.447854467Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Upstairs opts: Upstairs UUID: d8f70a36-7526-4921-ab90-a517a16c671a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.447893104Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Crucible stats registered with UUID: d8f70a36-7526-4921-ab90-a517a16c671a","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.447909082Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:51.447921861Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448263877Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448290635Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371} {"msg":"d8f70a36-7526-4921-ab90-a517a16c671a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448352379Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448371337Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"0","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448393315Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"0","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448407143Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"1","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448421222Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"1","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448438331Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"2","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448494635Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"2","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448524962Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.44855058Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448579117Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"ds_connection connected from Ok(127.0.0.1:42454)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448607754Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"io task","client":"2","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"ds_connection connected from Ok(127.0.0.1:38224)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448701395Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"io task","client":"1","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"ds_connection connected from Ok(127.0.0.1:59410)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.448738701Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"io task","client":"0","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.449340203Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"2","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.4493675Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"1","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.449382319Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"0","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.449965242Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"1","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450099499Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"0","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450169622Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"2","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450552715Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450667943Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"[0]R flush_numbers[0..12]: [83, 75, 79, 78, 76, 73, 80, 79, 84, 84, 84, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450712559Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"[0]R generation[0..12]: [44, 40, 43, 42, 41, 39, 44, 43, 45, 45, 45, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450730447Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450745136Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"[1]R flush_numbers[0..12]: [83, 75, 79, 78, 76, 73, 80, 79, 84, 84, 84, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450758674Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"[1]R generation[0..12]: [44, 40, 43, 42, 41, 39, 44, 43, 45, 45, 45, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450772133Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450785872Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"[2]R flush_numbers[0..12]: [83, 75, 79, 78, 76, 73, 80, 79, 84, 84, 84, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450799001Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"[2]R generation[0..12]: [44, 40, 43, 42, 41, 39, 44, 43, 45, 45, 45, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450812089Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450825768Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450838397Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450851305Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Next flush: 85","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450863844Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:10:51.450876693Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"":"downstairs","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450895171Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.45090706Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"0","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450920129Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"1","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450933217Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"client":"2","":"downstairs","session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450946506Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450958625Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"d8f70a36-7526-4921-ab90-a517a16c671a is now active with session: 06008ac9-4ab3-4f29-84d7-dd1ac0f59c93","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450971074Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450982723Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371,"session_id":"06008ac9-4ab3-4f29-84d7-dd1ac0f59c93"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:51.450994522Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 271 len 24576 data: 6 3 3 5 6 6 02/30 Read block 581 len 24576 03/30 Write block 163 len 20480 data: 7 8 5 4 4 04/30 Read block 194 len 24576 05/30 Read block 391 len 32768 06/30 Write block 531 len 4096 data: 8 07/30 Write block 580 len 40960 data: 6 6 7 7 7 8 5 3 4 5 08/30 Write block 532 len 16384 data: 7 8 4 5 09/30 Read block 445 len 36864 10/30 Write block 440 len 20480 data: 5 4 5 5 4 11/30 Read block 12 len 12288 12/30 Write block 488 len 28672 data: 5 5 4 4 4 5 5 13/30 Read block 540 len 24576 14/30 Read block 184 len 12288 15/30 Read block 124 len 24576 16/30 Write block 386 len 16384 data: 6 6 6 5 17/30 Flush 18/30 Write block 325 len 20480 data: 5 5 6 6 5 19/30 Write block 326 len 16384 data: 6 7 7 6 20/30 Flush 21/30 Flush 22/30 Write block 330 len 16384 data: 4 4 4 5 23/30 Write block 569 len 24576 data: 4 5 5 4 4 3 24/30 Read block 176 len 8192 25/30 Read block 27 len 28672 26/30 Read block 221 len 24576 27/30 Write block 518 len 28672 data: 6 6 7 5 4 5 6 28/30 Read block 299 len 40960 29/30 Read block 408 len 36864 30/30 Read block 429 len 32768 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 8 9 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:51.471093254Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1371} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 83 83 83 F F F 1 020-039 40 40 40 75 75 75 F F F 2 040-059 43 43 43 79 79 79 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 41 41 41 76 76 76 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 44 44 44 80 80 80 F F F 7 140-159 43 43 43 79 79 79 F F F 8 160-179 46 46 46 85 85 85 F F F 9 180-199 45 45 45 84 84 84 F F F 10 200-219 45 45 45 84 84 84 F F F 11 220-239 38 38 38 72 72 72 F F F 12 240-259 44 44 44 80 80 80 F F F 13 260-279 46 46 46 85 85 85 F F F 14 280-299 44 44 44 81 81 81 F F F 15 300-319 43 43 43 79 79 79 F F F 16 320-339 46 46 46 86 86 86 T T T 17 340-359 42 42 42 77 77 77 F F F 18 360-379 44 44 44 80 80 80 F F F 19 380-399 46 46 46 85 85 85 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 46 46 46 85 85 85 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 46 46 46 85 85 85 F F F 25 500-519 44 44 44 83 83 83 T T T 26 520-539 46 46 46 85 85 85 T T T 27 540-559 44 44 44 81 81 81 F F F 28 560-579 44 44 44 81 81 81 T T T 29 580-599 46 46 46 85 85 85 F F F Max gen: 46, Max flush: 86 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 83 83 83 F F F 1 020-039 40 40 40 75 75 75 F F F 2 040-059 43 43 43 79 79 79 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 41 41 41 76 76 76 F F F 5 100-119 39 39 39 73 73 73 F F F 6 120-139 44 44 44 80 80 80 F F F 7 140-159 43 43 43 79 79 79 F F F 8 160-179 46 46 46 85 85 85 F F F 9 180-199 45 45 45 84 84 84 F F F 10 200-219 45 45 45 84 84 84 F F F 11 220-239 38 38 38 72 72 72 F F F 12 240-259 44 44 44 80 80 80 F F F 13 260-279 46 46 46 85 85 85 F F F 14 280-299 44 44 44 81 81 81 F F F 15 300-319 43 43 43 79 79 79 F F F 16 320-339 47 47 47 87 87 87 F F F 17 340-359 42 42 42 77 77 77 F F F 18 360-379 44 44 44 80 80 80 F F F 19 380-399 46 46 46 85 85 85 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 46 46 46 85 85 85 F F F 23 460-479 39 39 39 73 73 73 F F F 24 480-499 46 46 46 85 85 85 F F F 25 500-519 47 47 47 87 87 87 F F F 26 520-539 47 47 47 87 87 87 F F F 27 540-559 44 44 44 81 81 81 F F F 28 560-579 47 47 47 87 87 87 F F F 29 580-599 46 46 46 85 85 85 F F F Max gen: 47, Max flush: 87 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.726463747Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.769477719Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.770051203Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.770093339Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.770108327Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.770122786Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379} {"msg":"Crucible b5680a45-c2e6-43ca-9aa1-33af72a267bc has session id: 90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.770663033Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Upstairs opts: Upstairs UUID: b5680a45-c2e6-43ca-9aa1-33af72a267bc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.770783352Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Crucible stats registered with UUID: b5680a45-c2e6-43ca-9aa1-33af72a267bc","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.770807649Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:55.770821258Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771238528Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771263475Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379} {"msg":"b5680a45-c2e6-43ca-9aa1-33af72a267bc active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771335598Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771353776Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"0","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771375384Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"0","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771392102Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"1","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.77141818Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"1","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771451777Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"2","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771481084Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"2","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771534289Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771572865Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771600832Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49019)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771627339Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"io task","client":"2","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64789)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771657956Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"io task","client":"0","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53440)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.771734429Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"io task","client":"1","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.772297564Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"0","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.77233765Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"2","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.772967059Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"0","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.773000566Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"2","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.773248502Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"1","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.773592468Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.773749883Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.773919986Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"1","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"[0]R flush_numbers[0..12]: [83, 75, 79, 78, 76, 73, 80, 79, 85, 84, 84, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.77439053Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"[0]R generation[0..12]: [44, 40, 43, 42, 41, 39, 44, 43, 46, 45, 45, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774418458Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774433766Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"[1]R flush_numbers[0..12]: [83, 75, 79, 78, 76, 73, 80, 79, 85, 84, 84, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774448095Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"[1]R generation[0..12]: [44, 40, 43, 42, 41, 39, 44, 43, 46, 45, 45, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774461634Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774475512Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"[2]R flush_numbers[0..12]: [83, 75, 79, 78, 76, 73, 80, 79, 85, 84, 84, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774489001Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"[2]R generation[0..12]: [44, 40, 43, 42, 41, 39, 44, 43, 46, 45, 45, 38]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.77450228Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:10:55.774519418Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774535496Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774548735Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774561444Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774574353Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"":"downstairs","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774587681Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.77459961Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"0","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774612899Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"1","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774625867Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"client":"2","":"downstairs","session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774638956Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774651755Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"b5680a45-c2e6-43ca-9aa1-33af72a267bc is now active with session: 90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774664394Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"session_id":"90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774676153Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379,"session_id":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "90f3c9a3-2fdb-4aa7-93d3-ff3932ade60f"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:10:55.774692111Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 495 len 24576 data: 5 4 4 5 5 5 02/30 Flush 03/30 Write block 346 len 4096 data: 6 04/30 Write block 261 len 32768 data: 4 5 5 6 7 7 6 6 05/30 Read block 592 len 16384 06/30 Read block 228 len 8192 07/30 Read block 273 len 20480 08/30 Write block 107 len 8192 data: 4 5 09/30 Write block 210 len 4096 data: 5 10/30 Write block 285 len 16384 data: 3 3 3 3 11/30 Write block 584 len 4096 data: 8 12/30 Read block 85 len 24576 13/30 Read block 470 len 4096 14/30 Write block 492 len 4096 data: 5 15/30 Write block 478 len 16384 data: 7 7 7 6 16/30 Read block 63 len 8192 17/30 Write block 549 len 32768 data: 3 5 5 5 5 6 5 4 18/30 Write block 235 len 40960 data: 5 4 4 5 5 6 6 6 6 6 19/30 Write block 144 len 16384 data: 4 4 4 3 20/30 Read block 434 len 24576 21/30 Write block 575 len 4096 data: 2 22/30 Read block 328 len 16384 23/30 Flush 24/30 Write block 346 len 24576 data: 7 7 7 5 5 5 25/30 Read block 346 len 32768 26/30 Write block 139 len 20480 data: 9 11 9 6 6 27/30 Write block 158 len 40960 data: 7 7 7 7 7 8 9 6 5 5 28/30 Write block 508 len 28672 data: 5 6 7 6 7 9 8 29/30 Write block 101 len 32768 data: 3 4 3 4 3 3 5 6 30/30 Read block 465 len 40960 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 7 5 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:10:55.792038772Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1379} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 83 83 83 F F F 1 020-039 40 40 40 75 75 75 F F F 2 040-059 43 43 43 79 79 79 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 41 41 41 76 76 76 F F F 5 100-119 48 48 48 89 89 89 T T T 6 120-139 44 44 44 80 80 80 T T T 7 140-159 48 48 48 89 89 89 T T T 8 160-179 46 46 46 85 85 85 T T T 9 180-199 45 45 45 84 84 84 F F F 10 200-219 48 48 48 89 89 89 F F F 11 220-239 48 48 48 89 89 89 F F F 12 240-259 48 48 48 89 89 89 F F F 13 260-279 48 48 48 89 89 89 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 43 43 43 79 79 79 F F F 16 320-339 47 47 47 87 87 87 F F F 17 340-359 48 48 48 89 89 89 T T T 18 360-379 44 44 44 80 80 80 F F F 19 380-399 46 46 46 85 85 85 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 46 46 46 85 85 85 F F F 23 460-479 48 48 48 89 89 89 F F F 24 480-499 48 48 48 89 89 89 F F F 25 500-519 48 48 48 88 88 88 T T T 26 520-539 47 47 47 87 87 87 F F F 27 540-559 48 48 48 89 89 89 F F F 28 560-579 48 48 48 89 89 89 F F F 29 580-599 48 48 48 89 89 89 F F F Max gen: 48, Max flush: 89 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 83 83 83 F F F 1 020-039 40 40 40 75 75 75 F F F 2 040-059 43 43 43 79 79 79 F F F 3 060-079 42 42 42 78 78 78 F F F 4 080-099 41 41 41 76 76 76 F F F 5 100-119 49 49 49 90 90 90 F F F 6 120-139 49 49 49 90 90 90 F F F 7 140-159 49 49 49 90 90 90 F F F 8 160-179 49 49 49 90 90 90 F F F 9 180-199 45 45 45 84 84 84 F F F 10 200-219 48 48 48 89 89 89 F F F 11 220-239 48 48 48 89 89 89 F F F 12 240-259 48 48 48 89 89 89 F F F 13 260-279 48 48 48 89 89 89 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 43 43 43 79 79 79 F F F 16 320-339 47 47 47 87 87 87 F F F 17 340-359 49 49 49 90 90 90 F F F 18 360-379 44 44 44 80 80 80 F F F 19 380-399 46 46 46 85 85 85 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 36 36 36 70 70 70 F F F 22 440-459 46 46 46 85 85 85 F F F 23 460-479 48 48 48 89 89 89 F F F 24 480-499 48 48 48 89 89 89 F F F 25 500-519 49 49 49 90 90 90 F F F 26 520-539 47 47 47 87 87 87 F F F 27 540-559 48 48 48 89 89 89 F F F 28 560-579 48 48 48 89 89 89 F F F 29 580-599 48 48 48 89 89 89 F F F Max gen: 49, Max flush: 90 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.031424461Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.07456592Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.075169802Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.075210607Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.075225356Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.075239815Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387} {"msg":"Crucible 0a1930fb-c076-4651-ae64-3db8d2772c5f has session id: 8c90ad0c-1fa8-418d-885e-81c364237051","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.075771213Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"Upstairs opts: Upstairs UUID: 0a1930fb-c076-4651-ae64-3db8d2772c5f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.075817568Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"Crucible stats registered with UUID: 0a1930fb-c076-4651-ae64-3db8d2772c5f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.075845096Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:00.075866454Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076285373Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.07631024Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387} {"msg":"0a1930fb-c076-4651-ae64-3db8d2772c5f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076376214Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076394512Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"0","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.07641804Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"0","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076432699Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"1","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076446897Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"1","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076465035Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"2","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076490213Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"2","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076514451Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076548867Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076575034Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"ds_connection connected from Ok(127.0.0.1:53484)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076599832Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"io task","client":"2","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"ds_connection connected from Ok(127.0.0.1:55047)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076630549Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"io task","client":"0","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"ds_connection connected from Ok(127.0.0.1:48812)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.076655967Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"io task","client":"1","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.077307813Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"2","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.077335451Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"0","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.077926233Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"1","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.07795436Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"2","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.077969539Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"0","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.078492968Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.078600497Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"1","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.07867021Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"[0]R flush_numbers[0..12]: [83, 75, 79, 78, 76, 90, 90, 90, 90, 84, 89, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079076741Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"[0]R generation[0..12]: [44, 40, 43, 42, 41, 49, 49, 49, 49, 45, 48, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079100139Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079114477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"[1]R flush_numbers[0..12]: [83, 75, 79, 78, 76, 90, 90, 90, 90, 84, 89, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079127936Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"[1]R generation[0..12]: [44, 40, 43, 42, 41, 49, 49, 49, 49, 45, 48, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079146014Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079160463Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"[2]R flush_numbers[0..12]: [83, 75, 79, 78, 76, 90, 90, 90, 90, 84, 89, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079174061Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"[2]R generation[0..12]: [44, 40, 43, 42, 41, 49, 49, 49, 49, 45, 48, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.0791875Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079201579Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079214248Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079227006Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"Wait for a query_work_queue command to finish before sending IO msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079248934Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079261963Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"":"downstairs","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079275042Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079286941Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"0","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079300099Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"1","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079312988Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"client":"2","":"downstairs","session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079326167Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079338316Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"0a1930fb-c076-4651-ae64-3db8d2772c5f is now active with session: 8c90ad0c-1fa8-418d-885e-81c364237051","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079353224Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079365213Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387,"session_id":"8c90ad0c-1fa8-418d-885e-81c364237051"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:00.079377072Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 593 len 28672 data: 3 3 2 2 2 2 2 02/30 Write block 505 len 16384 data: 9 8 7 6 03/30 Read block 127 len 20480 04/30 Read block 116 len 20480 05/30 Flush 06/30 Write block 377 len 12288 data: 3 5 6 07/30 Write block 375 len 16384 data: 2 3 4 6 08/30 Read block 250 len 4096 09/30 Write block 323 len 36864 data: 5 5 6 7 8 8 7 5 5 10/30 Write block 57 len 36864 data: 5 5 6 6 7 7 7 5 7 11/30 Flush 12/30 Write block 545 len 8192 data: 3 3 13/30 Write block 529 len 40960 data: 6 8 9 8 9 5 6 6 7 6 14/30 Read block 486 len 8192 15/30 Read block 230 len 36864 16/30 Read block 162 len 24576 17/30 Read block 594 len 12288 18/30 Write block 96 len 8192 data: 5 5 19/30 Write block 512 len 4096 data: 8 20/30 Write block 518 len 36864 data: 7 7 8 6 5 6 7 5 3 21/30 Write block 313 len 16384 data: 7 8 9 8 22/30 Write block 139 len 12288 data: 10 12 10 23/30 Write block 47 len 36864 data: 4 3 3 3 4 6 5 5 5 24/30 Write block 336 len 4096 data: 4 25/30 Read block 408 len 24576 26/30 Write block 462 len 40960 data: 4 4 4 4 4 4 3 4 4 4 27/30 Write block 421 len 32768 data: 3 4 4 4 4 4 4 3 28/30 Write block 44 len 36864 data: 5 3 3 5 4 4 4 5 7 29/30 Write block 571 len 32768 data: 6 5 5 4 3 3 4 5 30/30 Read block 540 len 40960 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 2 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 19 19 18 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:00.097939954Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1387} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 83 83 83 F F F 1 020-039 40 40 40 75 75 75 F F F 2 040-059 50 50 50 92 92 92 T T T 3 060-079 50 50 50 92 92 92 F F F 4 080-099 41 41 41 76 76 76 T T T 5 100-119 49 49 49 90 90 90 F F F 6 120-139 49 49 49 90 90 90 T T T 7 140-159 49 49 49 90 90 90 T T T 8 160-179 49 49 49 90 90 90 F F F 9 180-199 45 45 45 84 84 84 F F F 10 200-219 48 48 48 89 89 89 F F F 11 220-239 48 48 48 89 89 89 F F F 12 240-259 48 48 48 89 89 89 F F F 13 260-279 48 48 48 89 89 89 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 43 43 43 79 79 79 T T T 16 320-339 50 50 50 92 92 92 T T T 17 340-359 49 49 49 90 90 90 F F F 18 360-379 50 50 50 92 92 92 F F F 19 380-399 46 46 46 85 85 85 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 36 36 36 70 70 70 T T T 22 440-459 46 46 46 85 85 85 F F F 23 460-479 48 48 48 89 89 89 T T T 24 480-499 48 48 48 89 89 89 F F F 25 500-519 50 50 50 91 91 91 T T T 26 520-539 47 47 47 87 87 87 T T T 27 540-559 48 48 48 89 89 89 T T T 28 560-579 48 48 48 89 89 89 T T T 29 580-599 50 50 50 91 91 91 F F F Max gen: 50, Max flush: 92 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 83 83 83 F F F 1 020-039 40 40 40 75 75 75 F F F 2 040-059 51 51 51 93 93 93 F F F 3 060-079 50 50 50 92 92 92 F F F 4 080-099 51 51 51 93 93 93 F F F 5 100-119 49 49 49 90 90 90 F F F 6 120-139 51 51 51 93 93 93 F F F 7 140-159 51 51 51 93 93 93 F F F 8 160-179 49 49 49 90 90 90 F F F 9 180-199 45 45 45 84 84 84 F F F 10 200-219 48 48 48 89 89 89 F F F 11 220-239 48 48 48 89 89 89 F F F 12 240-259 48 48 48 89 89 89 F F F 13 260-279 48 48 48 89 89 89 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 51 51 51 93 93 93 F F F 16 320-339 51 51 51 93 93 93 F F F 17 340-359 49 49 49 90 90 90 F F F 18 360-379 50 50 50 92 92 92 F F F 19 380-399 46 46 46 85 85 85 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 51 51 51 93 93 93 F F F 22 440-459 46 46 46 85 85 85 F F F 23 460-479 51 51 51 93 93 93 F F F 24 480-499 48 48 48 89 89 89 F F F 25 500-519 51 51 51 93 93 93 F F F 26 520-539 51 51 51 93 93 93 F F F 27 540-559 51 51 51 93 93 93 F F F 28 560-579 51 51 51 93 93 93 F F F 29 580-599 50 50 50 91 91 91 F F F Max gen: 51, Max flush: 93 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.3662061Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:04.407871034Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.408259666Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.43922843Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.439789535Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.439824862Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.439838691Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.43985256Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395} {"msg":"Crucible 3ba5531c-aa21-4db4-b885-0cabfc498892 has session id: d9021df0-a9dd-4edc-a3fe-8c79eb82ea59","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.440371999Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Upstairs opts: Upstairs UUID: 3ba5531c-aa21-4db4-b885-0cabfc498892, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.440417365Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Crucible stats registered with UUID: 3ba5531c-aa21-4db4-b885-0cabfc498892","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.440436883Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:04.440453031Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.440848603Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.440883649Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395} {"msg":"3ba5531c-aa21-4db4-b885-0cabfc498892 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.440937324Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.440963661Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"0","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.440991249Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"0","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.441006277Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"1","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.441020836Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"1","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.44108249Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"2","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.441120506Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"2","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.441149113Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.44117904Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.441207708Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"ds_connection connected from Ok(127.0.0.1:33925)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.441235385Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"io task","client":"0","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"ds_connection connected from Ok(127.0.0.1:35379)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.441262972Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"io task","client":"1","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"ds_connection connected from Ok(127.0.0.1:53366)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.441343884Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"io task","client":"2","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.441874532Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"2","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.44190374Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"1","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.441975333Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"0","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.442444427Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"1","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.442473404Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"2","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.442596172Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"0","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.442965356Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443070466Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"[0]R flush_numbers[0..12]: [83, 75, 93, 92, 93, 90, 93, 93, 90, 84, 89, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443114562Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"[0]R generation[0..12]: [44, 40, 51, 50, 51, 49, 51, 51, 49, 45, 48, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.44313234Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443146889Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"[1]R flush_numbers[0..12]: [83, 75, 93, 92, 93, 90, 93, 93, 90, 84, 89, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443161357Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"[1]R generation[0..12]: [44, 40, 51, 50, 51, 49, 51, 51, 49, 45, 48, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443174796Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443188475Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"[2]R flush_numbers[0..12]: [83, 75, 93, 92, 93, 90, 93, 93, 90, 84, 89, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443202063Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"[2]R generation[0..12]: [44, 40, 51, 50, 51, 49, 51, 51, 49, 45, 48, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443215512Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443229351Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443245619Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443259698Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","Wait for a query_work_queue command to finish before sending IO pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443291665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443304473Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"":"downstairs","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443317452Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443329281Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"0","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.44334241Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"1","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443355428Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"client":"2","":"downstairs","session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443368997Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443381276Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"3ba5531c-aa21-4db4-b885-0cabfc498892 is now active with session: d9021df0-a9dd-4edc-a3fe-8c79eb82ea59","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443393965Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443405894Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395,"session_id":"d9021df0-a9dd-4edc-a3fe-8c79eb82ea59"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:04.443417852Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 154 len 4096 data: 2 02/30 Read block 195 len 32768 03/30 Write block 272 len 20480 data: 4 4 6 7 7 04/30 Write block 490 len 24576 data: 5 5 6 6 6 6 05/30 Read block 471 len 36864 06/30 Read block 174 len 40960 07/30 Write block 205 len 8192 data: 6 6 08/30 Read block 213 len 32768 09/30 Flush 10/30 Write block 131 len 32768 data: 4 4 4 4 5 6 9 9 11/30 Write block 383 len 4096 data: 7 12/30 Read block 316 len 4096 13/30 Read block 10 len 36864 14/30 Read block 368 len 24576 15/30 Write block 572 len 32768 data: 6 6 5 4 4 5 6 5 16/30 Read block 476 len 36864 17/30 Write block 350 len 8192 data: 6 6 18/30 Read block 319 len 32768 19/30 Read block 361 len 16384 20/30 Read block 193 len 40960 21/30 Read block 226 len 20480 22/30 Flush 23/30 Write block 167 len 16384 data: 6 7 6 5 24/30 Read block 206 len 32768 25/30 Read block 463 len 16384 26/30 Read block 203 len 20480 27/30 Write block 598 len 4096 data: 3 28/30 Write block 176 len 40960 data: 7 7 6 6 7 5 6 6 5 6 29/30 Write block 33 len 32768 data: 4 5 4 4 5 5 5 5 30/30 Write block 482 len 20480 data: 8 7 5 6 5 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 4 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:04.462430521Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1395} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 83 83 83 F F F 1 020-039 40 40 40 75 75 75 T T T 2 040-059 51 51 51 93 93 93 T T T 3 060-079 50 50 50 92 92 92 F F F 4 080-099 51 51 51 93 93 93 F F F 5 100-119 49 49 49 90 90 90 F F F 6 120-139 52 52 52 95 95 95 F F F 7 140-159 52 52 52 94 94 94 F F F 8 160-179 49 49 49 90 90 90 T T T 9 180-199 45 45 45 84 84 84 T T T 10 200-219 52 52 52 94 94 94 F F F 11 220-239 48 48 48 89 89 89 F F F 12 240-259 48 48 48 89 89 89 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 51 51 51 93 93 93 F F F 16 320-339 51 51 51 93 93 93 F F F 17 340-359 52 52 52 95 95 95 F F F 18 360-379 50 50 50 92 92 92 F F F 19 380-399 52 52 52 95 95 95 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 51 51 51 93 93 93 F F F 22 440-459 46 46 46 85 85 85 F F F 23 460-479 51 51 51 93 93 93 F F F 24 480-499 52 52 52 94 94 94 T T T 25 500-519 51 51 51 93 93 93 F F F 26 520-539 51 51 51 93 93 93 F F F 27 540-559 51 51 51 93 93 93 F F F 28 560-579 52 52 52 95 95 95 F F F 29 580-599 50 50 50 91 91 91 T T T Max gen: 52, Max flush: 95 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 83 83 83 F F F 1 020-039 53 53 53 96 96 96 F F F 2 040-059 53 53 53 96 96 96 F F F 3 060-079 50 50 50 92 92 92 F F F 4 080-099 51 51 51 93 93 93 F F F 5 100-119 49 49 49 90 90 90 F F F 6 120-139 52 52 52 95 95 95 F F F 7 140-159 52 52 52 94 94 94 F F F 8 160-179 53 53 53 96 96 96 F F F 9 180-199 53 53 53 96 96 96 F F F 10 200-219 52 52 52 94 94 94 F F F 11 220-239 48 48 48 89 89 89 F F F 12 240-259 48 48 48 89 89 89 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 48 48 48 89 89 89 F F F 15 300-319 51 51 51 93 93 93 F F F 16 320-339 51 51 51 93 93 93 F F F 17 340-359 52 52 52 95 95 95 F F F 18 360-379 50 50 50 92 92 92 F F F 19 380-399 52 52 52 95 95 95 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 51 51 51 93 93 93 F F F 22 440-459 46 46 46 85 85 85 F F F 23 460-479 51 51 51 93 93 93 F F F 24 480-499 53 53 53 96 96 96 F F F 25 500-519 51 51 51 93 93 93 F F F 26 520-539 51 51 51 93 93 93 F F F 27 540-559 51 51 51 93 93 93 F F F 28 560-579 52 52 52 95 95 95 F F F 29 580-599 53 53 53 96 96 96 F F F Max gen: 53, Max flush: 96 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.733789956Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:08.775287915Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.775703915Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.806370778Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.806932204Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.80696585Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.806985818Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.807003837Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403} {"msg":"Crucible 4084c533-07ca-4307-abe5-8bb32ec0d37c has session id: dbab2e91-f187-44a2-9532-a5a2b45b2ece","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.807554223Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Upstairs opts: Upstairs UUID: 4084c533-07ca-4307-abe5-8bb32ec0d37c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.807599028Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Crucible stats registered with UUID: 4084c533-07ca-4307-abe5-8bb32ec0d37c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.807626826Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:08.807648364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808050195Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808076632Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403} {"msg":"4084c533-07ca-4307-abe5-8bb32ec0d37c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808135256Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808152875Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"0","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808175842Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"0","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808190051Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"1","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808204Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"1","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808220608Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"2","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808245865Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"2","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808270343Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808297151Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808323868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"ds_connection connected from Ok(127.0.0.1:43305)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808449226Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"io task","client":"2","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"ds_connection connected from Ok(127.0.0.1:36239)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808554835Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"io task","client":"0","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"ds_connection connected from Ok(127.0.0.1:37516)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.808586312Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"io task","client":"1","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.809175385Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"2","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.809206432Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"1","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.80923145Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"0","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.809780916Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"2","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.809902654Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"1","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.809923382Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"0","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810365139Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810467329Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"[0]R flush_numbers[0..12]: [83, 96, 96, 92, 93, 90, 95, 94, 96, 96, 94, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810492687Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"[0]R generation[0..12]: [44, 53, 53, 50, 51, 49, 52, 52, 53, 53, 52, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810507875Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810522024Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"[1]R flush_numbers[0..12]: [83, 96, 96, 92, 93, 90, 95, 94, 96, 96, 94, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810535363Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"[1]R generation[0..12]: [44, 53, 53, 50, 51, 49, 52, 52, 53, 53, 52, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810552081Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810573139Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"[2]R flush_numbers[0..12]: [83, 96, 96, 92, 93, 90, 95, 94, 96, 96, 94, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810599176Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"[2]R generation[0..12]: [44, 53, 53, 50, 51, 49, 52, 52, 53, 53, 52, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810623364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:11:08.810638663Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810656741Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.8106699Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810682588Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810695177Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"":"downstairs","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810707956Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810719905Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"0","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810733643Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"1","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810746632Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"client":"2","":"downstairs","session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810759521Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.81077153Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"4084c533-07ca-4307-abe5-8bb32ec0d37c is now active with session: dbab2e91-f187-44a2-9532-a5a2b45b2ece","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810786708Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810798467Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403,"session_id":"dbab2e91-f187-44a2-9532-a5a2b45b2ece"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:08.810810456Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 80 len 24576 data: 4 5 5 5 4 5 02/30 Flush 03/30 Write block 19 len 16384 data: 3 2 3 5 04/30 Write block 315 len 32768 data: 10 9 7 6 2 3 4 6 05/30 Write block 202 len 8192 data: 3 5 06/30 Write block 464 len 12288 data: 5 5 5 07/30 Write block 552 len 24576 data: 6 6 7 6 5 4 08/30 Read block 566 len 20480 09/30 Flush 10/30 Write block 578 len 4096 data: 7 11/30 Write block 422 len 8192 data: 5 5 12/30 Write block 212 len 20480 data: 4 3 3 3 3 13/30 Read block 272 len 20480 14/30 Read block 39 len 4096 15/30 Read block 334 len 28672 16/30 Read block 398 len 28672 17/30 Flush 18/30 Write block 113 len 8192 data: 3 3 19/30 Write block 292 len 16384 data: 5 5 5 3 20/30 Write block 162 len 16384 data: 8 9 10 7 21/30 Write block 433 len 12288 data: 5 7 6 22/30 Flush 23/30 Write block 545 len 40960 data: 4 4 3 3 4 6 6 7 7 8 24/30 Read block 516 len 20480 25/30 Read block 588 len 24576 26/30 Read block 310 len 4096 27/30 Write block 436 len 40960 data: 7 7 7 6 6 5 6 6 5 5 28/30 Write block 223 len 4096 data: 4 29/30 Read block 91 len 12288 30/30 Read block 496 len 36864 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:08.828001922Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1403} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 98 98 98 F F F 1 020-039 54 54 54 98 98 98 F F F 2 040-059 53 53 53 96 96 96 F F F 3 060-079 50 50 50 92 92 92 F F F 4 080-099 54 54 54 97 97 97 F F F 5 100-119 54 54 54 100 100 100 F F F 6 120-139 52 52 52 95 95 95 F F F 7 140-159 52 52 52 94 94 94 F F F 8 160-179 54 54 54 100 100 100 F F F 9 180-199 53 53 53 96 96 96 F F F 10 200-219 54 54 54 99 99 99 F F F 11 220-239 48 48 48 89 89 89 T T T 12 240-259 48 48 48 89 89 89 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 54 54 54 100 100 100 F F F 15 300-319 54 54 54 98 98 98 F F F 16 320-339 54 54 54 98 98 98 F F F 17 340-359 52 52 52 95 95 95 F F F 18 360-379 50 50 50 92 92 92 F F F 19 380-399 52 52 52 95 95 95 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 54 54 54 100 100 100 T T T 22 440-459 46 46 46 85 85 85 T T T 23 460-479 54 54 54 98 98 98 F F F 24 480-499 53 53 53 96 96 96 F F F 25 500-519 51 51 51 93 93 93 F F F 26 520-539 51 51 51 93 93 93 F F F 27 540-559 54 54 54 98 98 98 T T T 28 560-579 54 54 54 99 99 99 F F F 29 580-599 53 53 53 96 96 96 F F F Max gen: 54, Max flush: 100 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 98 98 98 F F F 1 020-039 54 54 54 98 98 98 F F F 2 040-059 53 53 53 96 96 96 F F F 3 060-079 50 50 50 92 92 92 F F F 4 080-099 54 54 54 97 97 97 F F F 5 100-119 54 54 54 100 100 100 F F F 6 120-139 52 52 52 95 95 95 F F F 7 140-159 52 52 52 94 94 94 F F F 8 160-179 54 54 54 100 100 100 F F F 9 180-199 53 53 53 96 96 96 F F F 10 200-219 54 54 54 99 99 99 F F F 11 220-239 55 55 55 101 101 101 F F F 12 240-259 48 48 48 89 89 89 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 54 54 54 100 100 100 F F F 15 300-319 54 54 54 98 98 98 F F F 16 320-339 54 54 54 98 98 98 F F F 17 340-359 52 52 52 95 95 95 F F F 18 360-379 50 50 50 92 92 92 F F F 19 380-399 52 52 52 95 95 95 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 55 55 55 101 101 101 F F F 22 440-459 55 55 55 101 101 101 F F F 23 460-479 54 54 54 98 98 98 F F F 24 480-499 53 53 53 96 96 96 F F F 25 500-519 51 51 51 93 93 93 F F F 26 520-539 51 51 51 93 93 93 F F F 27 540-559 55 55 55 101 101 101 F F F 28 560-579 54 54 54 99 99 99 F F F 29 580-599 53 53 53 96 96 96 F F F Max gen: 55, Max flush: 101 Begin loop 28 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.082248715Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.126115463Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.126729204Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.126781419Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.126810396Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.126827764Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411} {"msg":"Crucible a391e173-054b-4ce9-bf2a-b8407abffc60 has session id: ca893ef2-8dab-4109-9e52-92fb9c4af95c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.127336724Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Upstairs opts: Upstairs UUID: a391e173-054b-4ce9-bf2a-b8407abffc60, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.127372291Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Crucible stats registered with UUID: a391e173-054b-4ce9-bf2a-b8407abffc60","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.12738697Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:13.127402628Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.127788231Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.127827877Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411} {"msg":"a391e173-054b-4ce9-bf2a-b8407abffc60 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.127882451Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.127908369Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"0","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.127931557Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"0","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.127945805Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"1","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.127959924Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"1","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.127976792Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"2","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.128006639Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"2","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.128031037Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.128055245Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.128079872Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37665)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.12810619Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"io task","client":"2","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"ds_connection connected from Ok(127.0.0.1:54829)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.128133567Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"io task","client":"0","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56612)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.128156765Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"io task","client":"1","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.128810571Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"2","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.128848957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"0","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.128878774Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"1","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.129451529Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"0","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.129480096Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"1","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.129495534Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"2","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130000495Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130039321Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"[0]R flush_numbers[0..12]: [98, 98, 96, 92, 97, 100, 95, 94, 100, 96, 99, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.13005665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"[0]R generation[0..12]: [54, 54, 53, 50, 54, 54, 52, 52, 54, 53, 54, 55]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130071388Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130085517Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"[1]R flush_numbers[0..12]: [98, 98, 96, 92, 97, 100, 95, 94, 100, 96, 99, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130099166Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"[1]R generation[0..12]: [54, 54, 53, 50, 54, 54, 52, 52, 54, 53, 54, 55]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130112824Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130126383Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"[2]R flush_numbers[0..12]: [98, 98, 96, 92, 97, 100, 95, 94, 100, 96, 99, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130139672Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"[2]R generation[0..12]: [54, 54, 53, 50, 54, 54, 52, 52, 54, 53, 54, 55]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.13015259Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130169569Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130192257Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130221854Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130239412Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130252381Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"":"downstairs","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.13026513Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130280188Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"0","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130293347Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"1","":"downstairs","session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130306006Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"client":"2","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130321934Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130334013Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"a391e173-054b-4ce9-bf2a-b8407abffc60 is now active with session: ca893ef2-8dab-4109-9e52-92fb9c4af95c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130346562Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.13035839Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411,"session_id":"ca893ef2-8dab-4109-9e52-92fb9c4af95c"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:13.130370309Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 182 len 24576 data: 7 7 6 7 5 6 02/30 Flush 03/30 Flush 04/30 Write block 592 len 24576 data: 3 4 4 3 3 3 05/30 Read block 114 len 36864 06/30 Read block 549 len 16384 07/30 Write block 78 len 28672 data: 4 5 5 6 6 6 5 08/30 Write block 393 len 28672 data: 6 3 3 3 3 4 4 09/30 Read block 271 len 40960 10/30 Write block 568 len 32768 data: 4 5 6 7 7 7 6 5 11/30 Write block 590 len 12288 data: 3 3 4 12/30 Read block 4 len 40960 13/30 Write block 352 len 40960 data: 5 6 5 4 3 4 7 6 6 5 14/30 Read block 264 len 20480 15/30 Read block 190 len 32768 16/30 Read block 128 len 36864 17/30 Read block 403 len 24576 18/30 Write block 554 len 24576 data: 9 7 6 5 4 4 19/30 Write block 75 len 16384 data: 4 4 4 5 20/30 Read block 144 len 12288 21/30 Write block 107 len 4096 data: 6 22/30 Flush 23/30 Write block 234 len 40960 data: 4 6 5 5 6 6 7 7 7 7 24/30 Read block 28 len 8192 25/30 Read block 184 len 16384 26/30 Write block 32 len 28672 data: 5 5 6 5 5 6 6 27/30 Write block 388 len 4096 data: 7 28/30 Read block 184 len 20480 29/30 Read block 372 len 28672 30/30 Read block 590 len 8192 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:13.151027828Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1411} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 98 98 98 F F F 1 020-039 54 54 54 98 98 98 T T T 2 040-059 53 53 53 96 96 96 F F F 3 060-079 56 56 56 104 104 104 F F F 4 080-099 56 56 56 104 104 104 F F F 5 100-119 56 56 56 104 104 104 F F F 6 120-139 52 52 52 95 95 95 F F F 7 140-159 52 52 52 94 94 94 F F F 8 160-179 54 54 54 100 100 100 F F F 9 180-199 56 56 56 102 102 102 F F F 10 200-219 54 54 54 99 99 99 F F F 11 220-239 55 55 55 101 101 101 T T T 12 240-259 48 48 48 89 89 89 T T T 13 260-279 52 52 52 94 94 94 F F F 14 280-299 54 54 54 100 100 100 F F F 15 300-319 54 54 54 98 98 98 F F F 16 320-339 54 54 54 98 98 98 F F F 17 340-359 56 56 56 104 104 104 F F F 18 360-379 56 56 56 104 104 104 F F F 19 380-399 56 56 56 104 104 104 T T T 20 400-419 43 43 43 79 79 79 F F F 21 420-439 55 55 55 101 101 101 F F F 22 440-459 55 55 55 101 101 101 F F F 23 460-479 54 54 54 98 98 98 F F F 24 480-499 53 53 53 96 96 96 F F F 25 500-519 51 51 51 93 93 93 F F F 26 520-539 51 51 51 93 93 93 F F F 27 540-559 56 56 56 104 104 104 F F F 28 560-579 56 56 56 104 104 104 F F F 29 580-599 56 56 56 104 104 104 F F F Max gen: 56, Max flush: 104 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 98 98 98 F F F 1 020-039 57 57 57 105 105 105 F F F 2 040-059 53 53 53 96 96 96 F F F 3 060-079 56 56 56 104 104 104 F F F 4 080-099 56 56 56 104 104 104 F F F 5 100-119 56 56 56 104 104 104 F F F 6 120-139 52 52 52 95 95 95 F F F 7 140-159 52 52 52 94 94 94 F F F 8 160-179 54 54 54 100 100 100 F F F 9 180-199 56 56 56 102 102 102 F F F 10 200-219 54 54 54 99 99 99 F F F 11 220-239 57 57 57 105 105 105 F F F 12 240-259 57 57 57 105 105 105 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 54 54 54 100 100 100 F F F 15 300-319 54 54 54 98 98 98 F F F 16 320-339 54 54 54 98 98 98 F F F 17 340-359 56 56 56 104 104 104 F F F 18 360-379 56 56 56 104 104 104 F F F 19 380-399 57 57 57 105 105 105 F F F 20 400-419 43 43 43 79 79 79 F F F 21 420-439 55 55 55 101 101 101 F F F 22 440-459 55 55 55 101 101 101 F F F 23 460-479 54 54 54 98 98 98 F F F 24 480-499 53 53 53 96 96 96 F F F 25 500-519 51 51 51 93 93 93 F F F 26 520-539 51 51 51 93 93 93 F F F 27 540-559 56 56 56 104 104 104 F F F 28 560-579 56 56 56 104 104 104 F F F 29 580-599 56 56 56 104 104 104 F F F Max gen: 57, Max flush: 105 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.377450258Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.420251561Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.420862221Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.420916666Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.420935284Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.420953113Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419} {"msg":"Crucible 2e265762-cd39-4cb2-b514-9757ab7fa114 has session id: 5f75b571-8ba8-413d-97ee-6cb185bfbc9b","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.421606279Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Upstairs opts: Upstairs UUID: 2e265762-cd39-4cb2-b514-9757ab7fa114, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.421663583Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Crucible stats registered with UUID: 2e265762-cd39-4cb2-b514-9757ab7fa114","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.421692281Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:17.421717768Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.42221097Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422242977Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419} {"msg":"2e265762-cd39-4cb2-b514-9757ab7fa114 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422295422Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422321819Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"0","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422349997Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"0","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422365755Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"1","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422380254Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"1","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422393142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"2","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.4224143Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"2","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422439848Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422464485Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422504751Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46001)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422533728Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"io task","client":"2","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42725)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422559096Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"io task","client":"1","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41481)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.422583814Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"io task","client":"0","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.423272687Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"0","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.423300994Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"1","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.423315963Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"2","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.423904825Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"1","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.423932043Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"2","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.423947001Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"0","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424503477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424527734Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"[0]R flush_numbers[0..12]: [98, 105, 96, 104, 104, 104, 95, 94, 100, 102, 99, 105]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424551632Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"[0]R generation[0..12]: [54, 57, 53, 56, 56, 56, 52, 52, 54, 56, 54, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424566231Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424580309Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"[1]R flush_numbers[0..12]: [98, 105, 96, 104, 104, 104, 95, 94, 100, 102, 99, 105]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424593748Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"[1]R generation[0..12]: [54, 57, 53, 56, 56, 56, 52, 52, 54, 56, 54, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424606897Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424620575Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"[2]R flush_numbers[0..12]: [98, 105, 96, 104, 104, 104, 95, 94, 100, 102, 99, 105]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424633914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"[2]R generation[0..12]: [54, 57, 53, 56, 56, 56, 52, 52, 54, 56, 54, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424647033Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424660712Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.42467683Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424690238Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424703007Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424722526Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"":"downstairs","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424735914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424747783Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"0","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424760892Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"1","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424773781Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"client":"2","":"downstairs","session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424786959Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424798938Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"2e265762-cd39-4cb2-b514-9757ab7fa114 is now active with session: 5f75b571-8ba8-413d-97ee-6cb185bfbc9b","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424811407Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424823076Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419,"session_id":"5f75b571-8ba8-413d-97ee-6cb185bfbc9b"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:17.424835064Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 169 len 32768 data: 7 6 6 8 8 8 9 8 02/30 Read block 171 len 8192 03/30 Read block 172 len 12288 04/30 Read block 204 len 28672 05/30 Write block 324 len 20480 data: 6 7 8 9 9 06/30 Read block 299 len 28672 07/30 Write block 368 len 20480 data: 5 4 4 4 2 08/30 Write block 197 len 16384 data: 4 3 3 3 09/30 Write block 416 len 16384 data: 5 4 3 3 10/30 Write block 300 len 8192 data: 6 6 11/30 Read block 49 len 40960 12/30 Flush 13/30 Write block 60 len 40960 data: 7 8 8 8 6 8 7 6 6 5 14/30 Write block 418 len 20480 data: 4 4 3 4 6 15/30 Write block 150 len 12288 data: 3 3 2 16/30 Write block 248 len 36864 data: 8 7 8 8 9 7 7 5 6 17/30 Write block 142 len 36864 data: 7 7 5 5 5 4 3 3 4 18/30 Write block 389 len 24576 data: 6 5 6 7 7 4 19/30 Write block 530 len 24576 data: 9 10 9 10 6 7 20/30 Read block 136 len 4096 21/30 Read block 12 len 28672 22/30 Flush 23/30 Write block 237 len 28672 data: 6 7 7 8 8 8 8 24/30 Flush 25/30 Write block 234 len 4096 data: 5 26/30 Read block 223 len 8192 27/30 Read block 414 len 8192 28/30 Read block 484 len 28672 29/30 Read block 535 len 40960 30/30 Write block 139 len 16384 data: 11 13 11 8 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:17.443180528Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1419} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 98 98 98 F F F 1 020-039 57 57 57 105 105 105 F F F 2 040-059 53 53 53 96 96 96 F F F 3 060-079 58 58 58 107 107 107 F F F 4 080-099 56 56 56 104 104 104 F F F 5 100-119 56 56 56 104 104 104 F F F 6 120-139 52 52 52 95 95 95 T T T 7 140-159 58 58 58 107 107 107 T T T 8 160-179 58 58 58 106 106 106 F F F 9 180-199 58 58 58 106 106 106 F F F 10 200-219 58 58 58 106 106 106 F F F 11 220-239 58 58 58 108 108 108 T T T 12 240-259 58 58 58 108 108 108 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 54 54 54 100 100 100 F F F 15 300-319 58 58 58 106 106 106 F F F 16 320-339 58 58 58 106 106 106 F F F 17 340-359 56 56 56 104 104 104 F F F 18 360-379 58 58 58 106 106 106 F F F 19 380-399 58 58 58 107 107 107 F F F 20 400-419 58 58 58 107 107 107 F F F 21 420-439 58 58 58 107 107 107 F F F 22 440-459 55 55 55 101 101 101 F F F 23 460-479 54 54 54 98 98 98 F F F 24 480-499 53 53 53 96 96 96 F F F 25 500-519 51 51 51 93 93 93 F F F 26 520-539 58 58 58 107 107 107 F F F 27 540-559 56 56 56 104 104 104 F F F 28 560-579 56 56 56 104 104 104 F F F 29 580-599 56 56 56 104 104 104 F F F Max gen: 58, Max flush: 108 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 98 98 98 F F F 1 020-039 57 57 57 105 105 105 F F F 2 040-059 53 53 53 96 96 96 F F F 3 060-079 58 58 58 107 107 107 F F F 4 080-099 56 56 56 104 104 104 F F F 5 100-119 56 56 56 104 104 104 F F F 6 120-139 59 59 59 109 109 109 F F F 7 140-159 59 59 59 109 109 109 F F F 8 160-179 58 58 58 106 106 106 F F F 9 180-199 58 58 58 106 106 106 F F F 10 200-219 58 58 58 106 106 106 F F F 11 220-239 59 59 59 109 109 109 F F F 12 240-259 58 58 58 108 108 108 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 54 54 54 100 100 100 F F F 15 300-319 58 58 58 106 106 106 F F F 16 320-339 58 58 58 106 106 106 F F F 17 340-359 56 56 56 104 104 104 F F F 18 360-379 58 58 58 106 106 106 F F F 19 380-399 58 58 58 107 107 107 F F F 20 400-419 58 58 58 107 107 107 F F F 21 420-439 58 58 58 107 107 107 F F F 22 440-459 55 55 55 101 101 101 F F F 23 460-479 54 54 54 98 98 98 F F F 24 480-499 53 53 53 96 96 96 F F F 25 500-519 51 51 51 93 93 93 F F F 26 520-539 58 58 58 107 107 107 F F F 27 540-559 56 56 56 104 104 104 F F F 28 560-579 56 56 56 104 104 104 F F F 29 580-599 56 56 56 104 104 104 F F F Max gen: 59, Max flush: 109 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.663778776Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.706522634Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.707172791Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.707221116Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.707239004Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.707256843Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427} {"msg":"Crucible 93a4da33-bce0-4627-8188-1f53ad61a7f0 has session id: 4935528b-d983-4468-8358-6aff71aa333a","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.707720337Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"Upstairs opts: Upstairs UUID: 93a4da33-bce0-4627-8188-1f53ad61a7f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.707763823Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"Crucible stats registered with UUID: 93a4da33-bce0-4627-8188-1f53ad61a7f0","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.707786721Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:21.707811618Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.708263865Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.708289462Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427} {"msg":"93a4da33-bce0-4627-8188-1f53ad61a7f0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.708352596Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.708371934Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"0","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.708395222Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"0","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.70840912Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"1","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.708426789Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"1","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.708445347Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"2","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.708472224Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"2","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.708496942Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.7085206Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.708542347Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46748)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.708564745Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"io task","client":"2","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50838)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.708591142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"io task","client":"0","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"ds_connection connected from Ok(127.0.0.1:41583)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.70862024Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"io task","client":"1","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.709256118Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"0","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.709285585Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"1","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.709877377Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"1","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.710067389Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"0","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.710451461Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.71056675Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.710632534Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"2","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711134575Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"2","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"[0]R flush_numbers[0..12]: [98, 105, 96, 107, 104, 104, 109, 109, 106, 106, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.71159635Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"[0]R generation[0..12]: [54, 57, 53, 58, 56, 56, 59, 59, 58, 58, 58, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711626027Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711645675Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"[1]R flush_numbers[0..12]: [98, 105, 96, 107, 104, 104, 109, 109, 106, 106, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711663343Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"[1]R generation[0..12]: [54, 57, 53, 58, 56, 56, 59, 59, 58, 58, 58, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711680162Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.71170128Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"[2]R flush_numbers[0..12]: [98, 105, 96, 107, 104, 104, 109, 109, 106, 106, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711719008Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"[2]R generation[0..12]: [54, 57, 53, 58, 56, 56, 59, 59, 58, 58, 58, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711746035Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711778292Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711809659Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711840836Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711872393Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.71190378Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"":"downstairs","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711936277Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"session_id":"4935528b-d983-4468-8358-6aff71aa333a"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.711969604Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"0","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.71200306Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"1","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.712033277Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"client":"2","":"downstairs","session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.712052325Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.712067504Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"93a4da33-bce0-4627-8188-1f53ad61a7f0 is now active with session: 4935528b-d983-4468-8358-6aff71aa333a","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.712083273Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.712102611Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427,"session_id":"4935528b-d983-4468-8358-6aff71aa333a"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:21.712117899Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 238 len 40960 data: 8 8 9 9 9 9 7 6 6 6 03/30 Read block 585 len 12288 04/30 Read block 147 len 4096 05/30 Read block 590 len 8192 06/30 Read block 27 len 32768 07/30 Write block 344 len 32768 data: 7 7 8 8 8 6 7 7 08/30 Read block 61 len 32768 09/30 Read block 419 len 4096 10/30 Write block 194 len 16384 data: 5 4 5 5 11/30 Write block 247 len 28672 data: 7 9 8 9 9 10 8 12/30 Write block 224 len 24576 data: 4 5 6 4 6 5 13/30 Read block 469 len 40960 14/30 Write block 223 len 32768 data: 5 5 6 7 5 7 6 4 15/30 Read block 83 len 32768 16/30 Read block 24 len 20480 17/30 Read block 598 len 8192 18/30 Read block 125 len 40960 19/30 Write block 208 len 32768 data: 6 5 6 4 5 4 4 4 20/30 Flush 21/30 Write block 407 len 28672 data: 3 4 4 4 4 5 5 22/30 Flush 23/30 Write block 79 len 24576 data: 6 6 7 7 7 6 24/30 Read block 94 len 36864 25/30 Write block 347 len 8192 data: 9 9 26/30 Write block 509 len 20480 data: 7 8 7 9 10 27/30 Read block 54 len 24576 28/30 Write block 113 len 32768 data: 4 4 4 4 5 4 4 4 29/30 Read block 82 len 12288 30/30 Read block 261 len 28672 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:21.732911285Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1427} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 98 98 98 F F F 1 020-039 57 57 57 105 105 105 F F F 2 040-059 53 53 53 96 96 96 F F F 3 060-079 58 58 58 107 107 107 T T T 4 080-099 56 56 56 104 104 104 T T T 5 100-119 56 56 56 104 104 104 T T T 6 120-139 59 59 59 109 109 109 T T T 7 140-159 59 59 59 109 109 109 F F F 8 160-179 58 58 58 106 106 106 F F F 9 180-199 60 60 60 111 111 111 F F F 10 200-219 60 60 60 111 111 111 F F F 11 220-239 60 60 60 111 111 111 F F F 12 240-259 60 60 60 111 111 111 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 54 54 54 100 100 100 F F F 15 300-319 58 58 58 106 106 106 F F F 16 320-339 58 58 58 106 106 106 F F F 17 340-359 60 60 60 111 111 111 T T T 18 360-379 58 58 58 106 106 106 F F F 19 380-399 58 58 58 107 107 107 F F F 20 400-419 60 60 60 112 112 112 F F F 21 420-439 58 58 58 107 107 107 F F F 22 440-459 55 55 55 101 101 101 F F F 23 460-479 54 54 54 98 98 98 F F F 24 480-499 53 53 53 96 96 96 F F F 25 500-519 51 51 51 93 93 93 T T T 26 520-539 58 58 58 107 107 107 F F F 27 540-559 56 56 56 104 104 104 F F F 28 560-579 56 56 56 104 104 104 F F F 29 580-599 56 56 56 104 104 104 F F F Max gen: 60, Max flush: 112 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 98 98 98 F F F 1 020-039 57 57 57 105 105 105 F F F 2 040-059 53 53 53 96 96 96 F F F 3 060-079 61 61 61 113 113 113 F F F 4 080-099 61 61 61 113 113 113 F F F 5 100-119 61 61 61 113 113 113 F F F 6 120-139 61 61 61 113 113 113 F F F 7 140-159 59 59 59 109 109 109 F F F 8 160-179 58 58 58 106 106 106 F F F 9 180-199 60 60 60 111 111 111 F F F 10 200-219 60 60 60 111 111 111 F F F 11 220-239 60 60 60 111 111 111 F F F 12 240-259 60 60 60 111 111 111 F F F 13 260-279 52 52 52 94 94 94 F F F 14 280-299 54 54 54 100 100 100 F F F 15 300-319 58 58 58 106 106 106 F F F 16 320-339 58 58 58 106 106 106 F F F 17 340-359 61 61 61 113 113 113 F F F 18 360-379 58 58 58 106 106 106 F F F 19 380-399 58 58 58 107 107 107 F F F 20 400-419 60 60 60 112 112 112 F F F 21 420-439 58 58 58 107 107 107 F F F 22 440-459 55 55 55 101 101 101 F F F 23 460-479 54 54 54 98 98 98 F F F 24 480-499 53 53 53 96 96 96 F F F 25 500-519 61 61 61 113 113 113 F F F 26 520-539 58 58 58 107 107 107 F F F 27 540-559 56 56 56 104 104 104 F F F 28 560-579 56 56 56 104 104 104 F F F 29 580-599 56 56 56 104 104 104 F F F Max gen: 61, Max flush: 113 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:25.966248193Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:26.007899818Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.008353413Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.039250935Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.039858895Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.039901181Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.039923909Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.039941958Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435} {"msg":"Crucible 755b887d-d5c0-4117-a869-6b8d6ba2a4a6 has session id: 0ba7a911-bd6a-48f6-bc29-289ef88ee3f1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.040519231Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Upstairs opts: Upstairs UUID: 755b887d-d5c0-4117-a869-6b8d6ba2a4a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.040557627Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Crucible stats registered with UUID: 755b887d-d5c0-4117-a869-6b8d6ba2a4a6","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.040573466Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:26.040586335Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.041054159Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.041077187Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435} {"msg":"755b887d-d5c0-4117-a869-6b8d6ba2a4a6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.04114721Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.041169628Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"0","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.041190456Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"0","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.041204295Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"1","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.041218833Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"1","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.041232162Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"2","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.04125308Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"2","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.041278897Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.041305075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.041329352Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:34504)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.04135794Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"io task","client":"2","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:43538)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.041386757Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"io task","client":"0","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:36735)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.041412004Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"io task","client":"1","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.042062781Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"1","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.042230135Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"2","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.04227144Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"0","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.042633425Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"1","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.042796209Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"2","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.042928447Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"0","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043187351Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043295901Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"[0]R flush_numbers[0..12]: [98, 105, 96, 113, 113, 113, 113, 109, 106, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043552606Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"[0]R generation[0..12]: [54, 57, 53, 61, 61, 61, 61, 59, 58, 60, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043572884Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043587663Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"[1]R flush_numbers[0..12]: [98, 105, 96, 113, 113, 113, 113, 109, 106, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043601431Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"[1]R generation[0..12]: [54, 57, 53, 61, 61, 61, 61, 59, 58, 60, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.04361463Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043628548Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"[2]R flush_numbers[0..12]: [98, 105, 96, 113, 113, 113, 113, 109, 106, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043642077Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"[2]R generation[0..12]: [54, 57, 53, 61, 61, 61, 61, 59, 58, 60, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043655116Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043668724Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043681243Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:11:26.043694252Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.04371194Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043724639Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"":"downstairs","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043740297Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043752176Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"0","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043765225Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"1","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043777914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"client":"2","":"downstairs","session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043790783Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043802711Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"755b887d-d5c0-4117-a869-6b8d6ba2a4a6 is now active with session: 0ba7a911-bd6a-48f6-bc29-289ef88ee3f1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.04381528Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043827119Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435,"session_id":"0ba7a911-bd6a-48f6-bc29-289ef88ee3f1"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:26.043839008Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 202 len 12288 data: 4 6 7 02/30 Write block 554 len 24576 data: 10 8 7 6 5 5 03/30 Read block 317 len 4096 04/30 Write block 403 len 32768 data: 8 5 6 6 4 5 5 5 05/30 Read block 483 len 16384 06/30 Write block 567 len 16384 data: 2 5 6 7 07/30 Read block 31 len 12288 08/30 Read block 205 len 36864 09/30 Read block 74 len 24576 10/30 Write block 256 len 8192 data: 7 5 11/30 Write block 505 len 32768 data: 10 9 8 7 8 9 8 10 12/30 Write block 519 len 16384 data: 8 9 7 6 13/30 Read block 463 len 36864 14/30 Read block 442 len 16384 15/30 Read block 521 len 12288 16/30 Write block 83 len 40960 data: 8 7 6 4 5 4 5 6 5 4 17/30 Read block 260 len 20480 18/30 Write block 416 len 8192 data: 6 5 19/30 Write block 407 len 8192 data: 5 6 20/30 Write block 567 len 32768 data: 3 6 7 8 8 8 8 7 21/30 Write block 265 len 12288 data: 8 8 7 22/30 Read block 335 len 40960 23/30 Flush 24/30 Write block 135 len 36864 data: 6 7 10 10 12 14 12 9 8 25/30 Read block 192 len 28672 26/30 Write block 303 len 24576 data: 4 3 3 5 6 7 27/30 Read block 224 len 28672 28/30 Read block 50 len 32768 29/30 Write block 310 len 32768 data: 9 9 8 8 9 11 10 8 30/30 Read block 162 len 32768 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:26.063888656Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1435} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 98 98 98 F F F 1 020-039 57 57 57 105 105 105 F F F 2 040-059 53 53 53 96 96 96 F F F 3 060-079 61 61 61 113 113 113 F F F 4 080-099 62 62 62 114 114 114 F F F 5 100-119 61 61 61 113 113 113 F F F 6 120-139 61 61 61 113 113 113 T T T 7 140-159 59 59 59 109 109 109 T T T 8 160-179 58 58 58 106 106 106 F F F 9 180-199 60 60 60 111 111 111 F F F 10 200-219 62 62 62 114 114 114 F F F 11 220-239 60 60 60 111 111 111 F F F 12 240-259 62 62 62 114 114 114 F F F 13 260-279 62 62 62 114 114 114 F F F 14 280-299 54 54 54 100 100 100 F F F 15 300-319 58 58 58 106 106 106 T T T 16 320-339 58 58 58 106 106 106 F F F 17 340-359 61 61 61 113 113 113 F F F 18 360-379 58 58 58 106 106 106 F F F 19 380-399 58 58 58 107 107 107 F F F 20 400-419 62 62 62 114 114 114 F F F 21 420-439 58 58 58 107 107 107 F F F 22 440-459 55 55 55 101 101 101 F F F 23 460-479 54 54 54 98 98 98 F F F 24 480-499 53 53 53 96 96 96 F F F 25 500-519 62 62 62 114 114 114 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 62 62 62 114 114 114 F F F 28 560-579 62 62 62 114 114 114 F F F 29 580-599 56 56 56 104 104 104 F F F Max gen: 62, Max flush: 114 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 98 98 98 F F F 1 020-039 57 57 57 105 105 105 F F F 2 040-059 53 53 53 96 96 96 F F F 3 060-079 61 61 61 113 113 113 F F F 4 080-099 62 62 62 114 114 114 F F F 5 100-119 61 61 61 113 113 113 F F F 6 120-139 63 63 63 115 115 115 F F F 7 140-159 63 63 63 115 115 115 F F F 8 160-179 58 58 58 106 106 106 F F F 9 180-199 60 60 60 111 111 111 F F F 10 200-219 62 62 62 114 114 114 F F F 11 220-239 60 60 60 111 111 111 F F F 12 240-259 62 62 62 114 114 114 F F F 13 260-279 62 62 62 114 114 114 F F F 14 280-299 54 54 54 100 100 100 F F F 15 300-319 63 63 63 115 115 115 F F F 16 320-339 58 58 58 106 106 106 F F F 17 340-359 61 61 61 113 113 113 F F F 18 360-379 58 58 58 106 106 106 F F F 19 380-399 58 58 58 107 107 107 F F F 20 400-419 62 62 62 114 114 114 F F F 21 420-439 58 58 58 107 107 107 F F F 22 440-459 55 55 55 101 101 101 F F F 23 460-479 54 54 54 98 98 98 F F F 24 480-499 53 53 53 96 96 96 F F F 25 500-519 62 62 62 114 114 114 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 62 62 62 114 114 114 F F F 28 560-579 62 62 62 114 114 114 F F F 29 580-599 56 56 56 104 104 104 F F F Max gen: 63, Max flush: 115 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.317072671Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.360264736Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.360929221Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.360977956Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.360997154Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.361015393Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443} {"msg":"Crucible 509d658c-0951-4de3-bcc9-456d59fe1172 has session id: 3cfbc521-873f-424e-bc0c-e2758a15a752","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.361474258Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Upstairs opts: Upstairs UUID: 509d658c-0951-4de3-bcc9-456d59fe1172, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.361530613Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Crucible stats registered with UUID: 509d658c-0951-4de3-bcc9-456d59fe1172","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.3615596Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:30.361585037Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362011046Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362045163Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443} {"msg":"509d658c-0951-4de3-bcc9-456d59fe1172 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362136723Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362162241Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"0","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362192328Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"0","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362207327Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"1","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362221565Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"1","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362234414Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"2","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362248143Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"2","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362269031Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362295918Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362317006Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"ds_connection connected from Ok(127.0.0.1:35029)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362341714Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"io task","client":"2","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"ds_connection connected from Ok(127.0.0.1:47444)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362367581Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"io task","client":"0","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"ds_connection connected from Ok(127.0.0.1:47781)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.362390699Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"io task","client":"1","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.363148845Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"0","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.363178692Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"1","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.363205859Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"2","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.363751386Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"2","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.363779803Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"0","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.363795142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"1","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364352548Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364377215Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"[0]R flush_numbers[0..12]: [98, 105, 96, 113, 114, 113, 115, 115, 106, 111, 114, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364404323Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"[0]R generation[0..12]: [54, 57, 53, 61, 62, 61, 63, 63, 58, 60, 62, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364419291Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.36443359Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"[1]R flush_numbers[0..12]: [98, 105, 96, 113, 114, 113, 115, 115, 106, 111, 114, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364453488Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"[1]R generation[0..12]: [54, 57, 53, 61, 62, 61, 63, 63, 58, 60, 62, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364467157Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364480695Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"[2]R flush_numbers[0..12]: [98, 105, 96, 113, 114, 113, 115, 115, 106, 111, 114, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364493874Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"[2]R generation[0..12]: [54, 57, 53, 61, 62, 61, 63, 63, 58, 60, 62, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364506753Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364520231Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.36453611Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364549349Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Next flush: 116","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:11:30.364562017Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364628011Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"":"downstairs","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364644999Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364656858Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"0","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364670007Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"1","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364682866Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"client":"2","":"downstairs","session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364695654Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364707543Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"509d658c-0951-4de3-bcc9-456d59fe1172 is now active with session: 3cfbc521-873f-424e-bc0c-e2758a15a752","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364720002Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.364731801Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443,"session_id":"3cfbc521-873f-424e-bc0c-e2758a15a752"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:30.36474368Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 330 len 40960 data: 6 6 5 6 5 4 5 5 5 5 03/30 Flush 04/30 Write block 447 len 20480 data: 5 4 2 2 2 05/30 Flush 06/30 Write block 342 len 8192 data: 7 7 07/30 Write block 7 len 20480 data: 6 6 5 7 7 08/30 Write block 493 len 16384 data: 7 7 7 5 09/30 Write block 261 len 32768 data: 5 6 6 7 9 9 8 7 10/30 Read block 29 len 4096 11/30 Read block 482 len 8192 12/30 Write block 331 len 16384 data: 7 6 7 6 13/30 Read block 468 len 4096 14/30 Write block 88 len 28672 data: 5 6 7 6 5 5 4 15/30 Write block 16 len 36864 data: 8 4 3 4 3 4 6 5 5 16/30 Read block 111 len 36864 17/30 Write block 334 len 16384 data: 7 5 6 6 18/30 Write block 433 len 32768 data: 6 8 7 8 8 8 7 7 19/30 Write block 423 len 4096 data: 6 20/30 Read block 545 len 40960 21/30 Write block 328 len 12288 data: 10 8 7 22/30 Write block 229 len 12288 data: 7 5 3 23/30 Read block 486 len 4096 24/30 Read block 6 len 8192 25/30 Read block 300 len 32768 26/30 Write block 289 len 12288 data: 4 4 4 27/30 Write block 584 len 16384 data: 9 9 6 4 28/30 Write block 515 len 36864 data: 7 8 7 8 9 10 8 7 7 29/30 Read block 457 len 20480 30/30 Write block 195 len 28672 data: 5 6 6 4 4 4 3 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 2 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:30.382957306Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1443} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 98 98 98 T T T 1 020-039 57 57 57 105 105 105 T T T 2 040-059 53 53 53 96 96 96 F F F 3 060-079 61 61 61 113 113 113 F F F 4 080-099 62 62 62 114 114 114 T T T 5 100-119 61 61 61 113 113 113 F F F 6 120-139 63 63 63 115 115 115 F F F 7 140-159 63 63 63 115 115 115 F F F 8 160-179 58 58 58 106 106 106 F F F 9 180-199 60 60 60 111 111 111 T T T 10 200-219 62 62 62 114 114 114 T T T 11 220-239 60 60 60 111 111 111 T T T 12 240-259 62 62 62 114 114 114 F F F 13 260-279 62 62 62 114 114 114 T T T 14 280-299 54 54 54 100 100 100 T T T 15 300-319 63 63 63 115 115 115 F F F 16 320-339 64 64 64 117 117 117 T T T 17 340-359 61 61 61 113 113 113 T T T 18 360-379 58 58 58 106 106 106 F F F 19 380-399 58 58 58 107 107 107 F F F 20 400-419 62 62 62 114 114 114 F F F 21 420-439 58 58 58 107 107 107 T T T 22 440-459 64 64 64 118 118 118 T T T 23 460-479 54 54 54 98 98 98 F F F 24 480-499 53 53 53 96 96 96 T T T 25 500-519 62 62 62 114 114 114 T T T 26 520-539 62 62 62 114 114 114 T T T 27 540-559 62 62 62 114 114 114 F F F 28 560-579 62 62 62 114 114 114 F F F 29 580-599 56 56 56 104 104 104 T T T Max gen: 64, Max flush: 118 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 119 119 119 F F F 1 020-039 65 65 65 119 119 119 F F F 2 040-059 53 53 53 96 96 96 F F F 3 060-079 61 61 61 113 113 113 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 61 61 61 113 113 113 F F F 6 120-139 63 63 63 115 115 115 F F F 7 140-159 63 63 63 115 115 115 F F F 8 160-179 58 58 58 106 106 106 F F F 9 180-199 65 65 65 119 119 119 F F F 10 200-219 65 65 65 119 119 119 F F F 11 220-239 65 65 65 119 119 119 F F F 12 240-259 62 62 62 114 114 114 F F F 13 260-279 65 65 65 119 119 119 F F F 14 280-299 65 65 65 119 119 119 F F F 15 300-319 63 63 63 115 115 115 F F F 16 320-339 65 65 65 119 119 119 F F F 17 340-359 65 65 65 119 119 119 F F F 18 360-379 58 58 58 106 106 106 F F F 19 380-399 58 58 58 107 107 107 F F F 20 400-419 62 62 62 114 114 114 F F F 21 420-439 65 65 65 119 119 119 F F F 22 440-459 65 65 65 119 119 119 F F F 23 460-479 54 54 54 98 98 98 F F F 24 480-499 65 65 65 119 119 119 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 62 62 62 114 114 114 F F F 28 560-579 62 62 62 114 114 114 F F F 29 580-599 65 65 65 119 119 119 F F F Max gen: 65, Max flush: 119 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.673191027Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.717120999Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.717761996Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.717804492Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.717819451Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.717837149Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451} {"msg":"Crucible 34898843-a202-4884-b4e4-3d16450e3df4 has session id: 2cdceb5d-c1ea-4929-980f-321be59aa8df","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.718382206Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Upstairs opts: Upstairs UUID: 34898843-a202-4884-b4e4-3d16450e3df4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.718428052Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Crucible stats registered with UUID: 34898843-a202-4884-b4e4-3d16450e3df4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.718451919Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:44.718472027Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.718941362Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.718967379Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451} {"msg":"34898843-a202-4884-b4e4-3d16450e3df4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.719025444Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.719051571Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"0","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.719075499Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"0","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.719089827Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"1","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.719104016Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"1","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.719117275Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"2","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.719130983Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"2","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.719148372Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.71916313Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.719179589Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"ds_connection connected from Ok(127.0.0.1:48696)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.719221485Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"ds_connection connected from Ok(127.0.0.1:43035)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.719388188Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"ds_connection connected from Ok(127.0.0.1:54573)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.719428514Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.720021596Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"2","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.720066842Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"0","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.720098679Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"1","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.720573193Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"2","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.720648915Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"1","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.720867234Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"0","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.72112112Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721239868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"[0]R flush_numbers[0..12]: [119, 119, 96, 113, 119, 113, 115, 115, 106, 119, 119, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721465346Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"[0]R generation[0..12]: [65, 65, 53, 61, 65, 61, 63, 63, 58, 65, 65, 65]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721488564Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721503582Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"[1]R flush_numbers[0..12]: [119, 119, 96, 113, 119, 113, 115, 115, 106, 119, 119, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721517201Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"[1]R generation[0..12]: [65, 65, 53, 61, 65, 61, 63, 63, 58, 65, 65, 65]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.72153032Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721543898Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"[2]R flush_numbers[0..12]: [119, 119, 96, 113, 119, 113, 115, 115, 106, 119, 119, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721557297Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"[2]R generation[0..12]: [65, 65, 53, 61, 65, 61, 63, 63, 58, 65, 65, 65]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721570636Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721584104Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721596733Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Generation requested: 66 >= found:66","vWait for a query_work_queue command to finish before sending IO ":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721616941Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721641299Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721666036Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"":"downstairs","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721691224Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721715342Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"0","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721745229Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"1","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721773166Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"client":"2","":"downstairs","session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721799743Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721823521Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"34898843-a202-4884-b4e4-3d16450e3df4 is now active with session: 2cdceb5d-c1ea-4929-980f-321be59aa8df","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721841019Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:44.721853048Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451,"session_id":"2cdceb5d-c1ea-4929-980f-321be59aa8df"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-28T17:11:44.721865097Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 449 len 24576 data: 3 3 3 3 4 4 03/30 Read block 107 len 32768 04/30 Write block 104 len 28672 data: 5 4 4 7 7 5 3 05/30 Write block 576 len 40960 data: 5 6 8 6 7 7 8 8 10 10 06/30 Write block 177 len 40960 data: 8 7 7 8 6 8 8 7 8 6 07/30 Write block 414 len 24576 data: 5 5 7 6 5 5 08/30 Write block 335 len 20480 data: 6 7 7 6 6 09/30 Flush 10/30 Write block 108 len 36864 data: 8 6 4 3 3 5 5 5 5 11/30 Read block 314 len 4096 12/30 Read block 301 len 16384 13/30 Read block 257 len 8192 14/30 Write block 333 len 8192 data: 8 8 15/30 Flush 16/30 Write block 277 len 36864 data: 6 7 7 5 4 4 3 2 4 17/30 Read block 591 len 8192 18/30 Write block 589 len 8192 data: 6 4 19/30 Read block 55 len 24576 20/30 Write block 129 len 20480 data: 2 2 5 5 5 21/30 Flush 22/30 Write block 460 len 12288 data: 4 3 5 23/30 Read block 158 len 36864 24/30 Write block 110 len 20480 data: 5 4 4 6 6 25/30 Write block 114 len 16384 data: 7 6 6 6 26/30 Write block 460 len 32768 data: 5 4 6 5 6 6 6 5 27/30 Write block 138 len 36864 data: 11 13 15 13 10 9 6 6 6 28/30 Write block 40 len 4096 data: 6 29/30 Read block 326 len 24576 30/30 Write block 13 len 32768 data: 7 6 7 9 5 4 5 4 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 3 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:44.740147307Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1451} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 119 119 119 T T T 1 020-039 65 65 65 119 119 119 T T T 2 040-059 53 53 53 96 96 96 T T T 3 060-079 61 61 61 113 113 113 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 66 66 66 122 122 122 T T T 6 120-139 66 66 66 123 123 123 T T T 7 140-159 63 63 63 115 115 115 T T T 8 160-179 66 66 66 121 121 121 F F F 9 180-199 66 66 66 121 121 121 F F F 10 200-219 65 65 65 119 119 119 F F F 11 220-239 65 65 65 119 119 119 F F F 12 240-259 62 62 62 114 114 114 F F F 13 260-279 66 66 66 123 123 123 F F F 14 280-299 66 66 66 123 123 123 F F F 15 300-319 63 63 63 115 115 115 F F F 16 320-339 66 66 66 122 122 122 F F F 17 340-359 65 65 65 119 119 119 F F F 18 360-379 58 58 58 106 106 106 F F F 19 380-399 58 58 58 107 107 107 F F F 20 400-419 66 66 66 121 121 121 F F F 21 420-439 65 65 65 119 119 119 F F F 22 440-459 66 66 66 121 121 121 F F F 23 460-479 54 54 54 98 98 98 T T T 24 480-499 65 65 65 119 119 119 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 62 62 62 114 114 114 F F F 28 560-579 66 66 66 121 121 121 F F F 29 580-599 66 66 66 123 123 123 F F F Max gen: 66, Max flush: 123 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 124 124 124 F F F 1 020-039 67 67 67 124 124 124 F F F 2 040-059 67 67 67 124 124 124 F F F 3 060-079 61 61 61 113 113 113 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 67 67 67 124 124 124 F F F 6 120-139 67 67 67 124 124 124 F F F 7 140-159 67 67 67 124 124 124 F F F 8 160-179 66 66 66 121 121 121 F F F 9 180-199 66 66 66 121 121 121 F F F 10 200-219 65 65 65 119 119 119 F F F 11 220-239 65 65 65 119 119 119 F F F 12 240-259 62 62 62 114 114 114 F F F 13 260-279 66 66 66 123 123 123 F F F 14 280-299 66 66 66 123 123 123 F F F 15 300-319 63 63 63 115 115 115 F F F 16 320-339 66 66 66 122 122 122 F F F 17 340-359 65 65 65 119 119 119 F F F 18 360-379 58 58 58 106 106 106 F F F 19 380-399 58 58 58 107 107 107 F F F 20 400-419 66 66 66 121 121 121 F F F 21 420-439 65 65 65 119 119 119 F F F 22 440-459 66 66 66 121 121 121 F F F 23 460-479 67 67 67 124 124 124 F F F 24 480-499 65 65 65 119 119 119 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 62 62 62 114 114 114 F F F 28 560-579 66 66 66 121 121 121 F F F 29 580-599 66 66 66 123 123 123 F F F Max gen: 67, Max flush: 124 Begin loop 34 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:48.990095407Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:49.032165321Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.032573911Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.063673863Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.064333509Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.064366925Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.064380624Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.064394872Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459} {"msg":"Crucible c8107659-c1e2-434c-9989-16956aa0444f has session id: 71527abe-79aa-4531-a540-b76f99f8fb24","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.064894974Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"Upstairs opts: Upstairs UUID: c8107659-c1e2-434c-9989-16956aa0444f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.064943879Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"Crucible stats registered with UUID: c8107659-c1e2-434c-9989-16956aa0444f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.064964407Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:49.064978016Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065329021Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065353499Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459} {"msg":"c8107659-c1e2-434c-9989-16956aa0444f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065419013Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065437861Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"0","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065459899Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"0","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065473347Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"1","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065487546Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"1","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065500645Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"2","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065515863Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"2","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065528942Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065542301Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.06555549Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"ds_connection connected from Ok(127.0.0.1:34000)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065630692Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"ds_connection connected from Ok(127.0.0.1:58727)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.06565151Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"ds_connection connected from Ok(127.0.0.1:35885)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.065704475Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.066432454Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"1","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.066467351Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"2","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.066501367Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"0","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.067097229Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"1","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.067127796Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"2","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.067155324Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"0","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.067674963Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.06770946Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"[0]R flush_numbers[0..12]: [124, 124, 124, 113, 119, 124, 124, 124, 121, 121, 119, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.067737137Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"[0]R generation[0..12]: [67, 67, 67, 61, 65, 67, 67, 67, 66, 66, 65, 65]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.067762155Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.067786602Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"[1]R flush_numbers[0..12]: [124, 124, 124, 113, 119, 124, 124, 124, 121, 121, 119, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.06780975Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"[1]R generation[0..12]: [67, 67, 67, 61, 65, 67, 67, 67, 66, 66, 65, 65]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.067834468Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.067860965Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"[2]R flush_numbers[0..12]: [124, 124, 124, 113, 119, 124, 124, 124, 121, 121, 119, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.067885253Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"[2]R generation[0..12]: [67, 67, 67, 61, 65, 67, 67, 67, 66, 66, 65, 65]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.06791324Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.067941527Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.067973324Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.068019819Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.068050527Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.068077274Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"":"downstairs","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.068103411Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.068127229Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"0","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"level":30,"time":"2025-01-28T17:11:49.068156976Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"1","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.068182724Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"client":"2","":"downstairs","session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.068208011Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.06822193Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"c8107659-c1e2-434c-9989-16956aa0444f is now active with session: 71527abe-79aa-4531-a540-b76f99f8fb24","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.068235129Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.068247127Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459,"session_id":"71527abe-79aa-4531-a540-b76f99f8fb24"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:49.068259086Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 342 len 32768 data: 8 8 8 8 9 10 10 7 02/30 Read block 472 len 12288 03/30 Write block 65 len 40960 data: 9 8 7 7 6 4 6 5 4 4 04/30 Read block 380 len 24576 05/30 Write block 220 len 32768 data: 3 4 4 6 6 7 8 6 06/30 Read block 453 len 32768 07/30 Read block 456 len 40960 08/30 Read block 504 len 28672 09/30 Read block 560 len 20480 10/30 Flush 11/30 Flush 12/30 Write block 256 len 12288 data: 8 6 5 13/30 Read block 7 len 24576 14/30 Write block 258 len 40960 data: 6 4 4 6 7 7 8 10 10 9 15/30 Write block 179 len 20480 data: 8 9 7 9 9 16/30 Read block 575 len 12288 17/30 Read block 329 len 8192 18/30 Read block 71 len 4096 19/30 Read block 320 len 20480 20/30 Write block 190 len 16384 data: 6 6 6 6 21/30 Flush 22/30 Write block 577 len 24576 data: 7 9 7 8 8 9 23/30 Write block 258 len 24576 data: 7 5 5 7 8 8 24/30 Read block 161 len 20480 25/30 Read block 88 len 4096 26/30 Read block 193 len 16384 27/30 Read block 470 len 20480 28/30 Write block 144 len 4096 data: 7 29/30 Read block 250 len 16384 30/30 Write block 556 len 32768 data: 8 7 6 6 4 3 4 3 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:49.088155659Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1459} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 124 124 124 F F F 1 020-039 67 67 67 124 124 124 F F F 2 040-059 67 67 67 124 124 124 F F F 3 060-079 68 68 68 125 125 125 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 67 67 67 124 124 124 F F F 6 120-139 67 67 67 124 124 124 F F F 7 140-159 67 67 67 124 124 124 T T T 8 160-179 68 68 68 127 127 127 F F F 9 180-199 68 68 68 127 127 127 F F F 10 200-219 65 65 65 119 119 119 F F F 11 220-239 68 68 68 125 125 125 F F F 12 240-259 68 68 68 127 127 127 T T T 13 260-279 68 68 68 127 127 127 T T T 14 280-299 66 66 66 123 123 123 F F F 15 300-319 63 63 63 115 115 115 F F F 16 320-339 66 66 66 122 122 122 F F F 17 340-359 68 68 68 125 125 125 F F F 18 360-379 58 58 58 106 106 106 F F F 19 380-399 58 58 58 107 107 107 F F F 20 400-419 66 66 66 121 121 121 F F F 21 420-439 65 65 65 119 119 119 F F F 22 440-459 66 66 66 121 121 121 F F F 23 460-479 67 67 67 124 124 124 F F F 24 480-499 65 65 65 119 119 119 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 62 62 62 114 114 114 T T T 28 560-579 66 66 66 121 121 121 T T T 29 580-599 66 66 66 123 123 123 T T T Max gen: 68, Max flush: 127 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 124 124 124 F F F 1 020-039 67 67 67 124 124 124 F F F 2 040-059 67 67 67 124 124 124 F F F 3 060-079 68 68 68 125 125 125 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 67 67 67 124 124 124 F F F 6 120-139 67 67 67 124 124 124 F F F 7 140-159 69 69 69 128 128 128 F F F 8 160-179 68 68 68 127 127 127 F F F 9 180-199 68 68 68 127 127 127 F F F 10 200-219 65 65 65 119 119 119 F F F 11 220-239 68 68 68 125 125 125 F F F 12 240-259 69 69 69 128 128 128 F F F 13 260-279 69 69 69 128 128 128 F F F 14 280-299 66 66 66 123 123 123 F F F 15 300-319 63 63 63 115 115 115 F F F 16 320-339 66 66 66 122 122 122 F F F 17 340-359 68 68 68 125 125 125 F F F 18 360-379 58 58 58 106 106 106 F F F 19 380-399 58 58 58 107 107 107 F F F 20 400-419 66 66 66 121 121 121 F F F 21 420-439 65 65 65 119 119 119 F F F 22 440-459 66 66 66 121 121 121 F F F 23 460-479 67 67 67 124 124 124 F F F 24 480-499 65 65 65 119 119 119 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 69 69 69 128 128 128 F F F 28 560-579 69 69 69 128 128 128 F F F 29 580-599 69 69 69 128 128 128 F F F Max gen: 69, Max flush: 128 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.351596976Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:53.393443141Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.393828944Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.424759272Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.42549998Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.425539116Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.425554195Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.425573053Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467} {"msg":"Crucible b92f0a20-257c-4bf5-9ffb-bda1fbd60cba has session id: d4d76299-be39-44de-ad05-2c4fa065c2aa","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.426151067Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Upstairs opts: Upstairs UUID: b92f0a20-257c-4bf5-9ffb-bda1fbd60cba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.4262228Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Crucible stats registered with UUID: b92f0a20-257c-4bf5-9ffb-bda1fbd60cba","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.426254477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:53.426273945Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.426658417Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.426691364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467} {"msg":"b92f0a20-257c-4bf5-9ffb-bda1fbd60cba active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.426748058Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.426773956Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"0","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.426797594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"0","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.426811972Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"1","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.426826051Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"1","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.42683905Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"2","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.426929181Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"2","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.426948099Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.427102284Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.427135211Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:33815)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.427162998Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:63804)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.427186846Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:53298)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.427211453Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.427602905Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"2","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.427641631Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"1","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.427733323Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"0","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.428178619Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"2","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.428207387Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"1","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.428356192Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"0","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.428729376Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.428833815Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"[0]R flush_numbers[0..12]: [124, 124, 124, 125, 119, 124, 124, 128, 127, 127, 119, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.428871972Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"[0]R generation[0..12]: [67, 67, 67, 68, 65, 67, 67, 69, 68, 68, 65, 68]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.42889324Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.428908408Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"[1]R flush_numbers[0..12]: [124, 124, 124, 125, 119, 124, 124, 128, 127, 127, 119, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.428922457Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"[1]R generation[0..12]: [67, 67, 67, 68, 65, 67, 67, 69, 68, 68, 65, 68]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.428935985Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.428949634Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"[2]R flush_numbers[0..12]: [124, 124, 124, 125, 119, 124, 124, 128, 127, 127, 119, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.428963503Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"[2]R generation[0..12]: [67, 67, 67, 68, 65, 67, 67, 69, 68, 68, 65, 68]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.428976912Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.42899397Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.429017338Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.429047075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.429063933Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.429077082Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"":"downstairs","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.42909001Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.429101879Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"0","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.429117668Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"1","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-28T17:11:53.429130617Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"client":"2","":"downstairs","session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.429146695Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.429158794Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"b92f0a20-257c-4bf5-9ffb-bda1fbd60cba is now active with session: d4d76299-be39-44de-ad05-2c4fa065c2aa","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.429171313Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.429182932Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467,"session_id":"d4d76299-be39-44de-ad05-2c4fa065c2aa"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:53.42919484Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 8192 data: 6 9 02/30 Write block 487 len 40960 data: 5 6 6 6 6 7 8 8 8 6 03/30 Write block 544 len 8192 data: 2 5 04/30 Write block 290 len 12288 data: 5 5 6 05/30 Read block 446 len 36864 06/30 Write block 213 len 4096 data: 5 07/30 Read block 59 len 4096 08/30 Write block 34 len 24576 data: 7 6 6 7 7 6 09/30 Write block 121 len 24576 data: 3 3 3 2 2 2 10/30 Write block 549 len 40960 data: 5 7 7 8 8 11 9 9 8 7 11/30 Read block 139 len 4096 12/30 Write block 555 len 4096 data: 10 13/30 Write block 126 len 12288 data: 3 2 2 14/30 Write block 309 len 28672 data: 8 10 10 9 9 10 12 15/30 Write block 433 len 4096 data: 7 16/30 Write block 582 len 24576 data: 10 9 11 11 7 5 17/30 Write block 168 len 36864 data: 8 8 7 7 9 9 9 10 9 18/30 Write block 8 len 36864 data: 7 6 8 8 7 8 7 8 10 19/30 Write block 72 len 20480 data: 6 5 5 5 5 20/30 Flush 21/30 Write block 180 len 40960 data: 10 8 10 10 8 9 7 7 5 6 22/30 Write block 214 len 28672 data: 5 5 4 3 2 3 4 23/30 Write block 42 len 24576 data: 4 5 6 4 4 6 24/30 Read block 530 len 24576 25/30 Read block 582 len 32768 26/30 Write block 209 len 40960 data: 6 7 5 6 6 6 6 5 4 3 27/30 Write block 51 len 20480 data: 6 8 6 6 6 28/30 Write block 64 len 16384 data: 7 10 9 8 29/30 Write block 379 len 24576 data: 7 6 7 7 8 7 30/30 Write block 339 len 24576 data: 7 6 7 10 9 9 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 5 Done Sent Sent false 1027 Acked Write 4 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 4 4 11 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:53.445163775Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1467} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 129 129 129 F F F 1 020-039 70 70 70 129 129 129 F F F 2 040-059 67 67 67 124 124 124 T T T 3 060-079 70 70 70 129 129 129 T T T 4 080-099 65 65 65 119 119 119 F F F 5 100-119 67 67 67 124 124 124 F F F 6 120-139 70 70 70 129 129 129 F F F 7 140-159 69 69 69 128 128 128 F F F 8 160-179 70 70 70 129 129 129 F F F 9 180-199 68 68 68 127 127 127 T T T 10 200-219 70 70 70 129 129 129 T T T 11 220-239 68 68 68 125 125 125 T T T 12 240-259 69 69 69 128 128 128 F F F 13 260-279 69 69 69 128 128 128 F F F 14 280-299 70 70 70 129 129 129 F F F 15 300-319 70 70 70 129 129 129 F F F 16 320-339 66 66 66 122 122 122 T T T 17 340-359 70 70 70 129 129 129 T T T 18 360-379 58 58 58 106 106 106 T T T 19 380-399 58 58 58 107 107 107 T T T 20 400-419 66 66 66 121 121 121 F F F 21 420-439 70 70 70 129 129 129 F F F 22 440-459 66 66 66 121 121 121 F F F 23 460-479 67 67 67 124 124 124 F F F 24 480-499 70 70 70 129 129 129 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 70 70 70 129 129 129 F F F 28 560-579 69 69 69 128 128 128 F F F 29 580-599 70 70 70 129 129 129 F F F Max gen: 70, Max flush: 129 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 129 129 129 F F F 1 020-039 70 70 70 129 129 129 F F F 2 040-059 71 71 71 130 130 130 F F F 3 060-079 71 71 71 130 130 130 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 67 67 67 124 124 124 F F F 6 120-139 70 70 70 129 129 129 F F F 7 140-159 69 69 69 128 128 128 F F F 8 160-179 70 70 70 129 129 129 F F F 9 180-199 71 71 71 130 130 130 F F F 10 200-219 71 71 71 130 130 130 F F F 11 220-239 71 71 71 130 130 130 F F F 12 240-259 69 69 69 128 128 128 F F F 13 260-279 69 69 69 128 128 128 F F F 14 280-299 70 70 70 129 129 129 F F F 15 300-319 70 70 70 129 129 129 F F F 16 320-339 71 71 71 130 130 130 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 71 71 71 130 130 130 F F F 19 380-399 71 71 71 130 130 130 F F F 20 400-419 66 66 66 121 121 121 F F F 21 420-439 70 70 70 129 129 129 F F F 22 440-459 66 66 66 121 121 121 F F F 23 460-479 67 67 67 124 124 124 F F F 24 480-499 70 70 70 129 129 129 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 70 70 70 129 129 129 F F F 28 560-579 69 69 69 128 128 128 F F F 29 580-599 70 70 70 129 129 129 F F F Max gen: 71, Max flush: 130 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.727848179Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:57.76942362Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.769815792Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.800610053Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.801170928Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.801205645Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.801219364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.801233613Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475} {"msg":"Crucible a47516c5-d60d-4c88-9dca-5998bab1019b has session id: 9882b20c-fadd-4569-9807-2159d777de15","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.801783739Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Upstairs opts: Upstairs UUID: a47516c5-d60d-4c88-9dca-5998bab1019b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.801819565Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Crucible stats registered with UUID: a47516c5-d60d-4c88-9dca-5998bab1019b","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.801835334Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:57.801848642Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802247744Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802272471Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475} {"msg":"a47516c5-d60d-4c88-9dca-5998bab1019b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802334655Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802352753Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"0","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802374911Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"0","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.80238836Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"1","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802402349Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"1","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802419017Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"2","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802498039Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"2","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802525187Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802552684Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802579042Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"ds_connection connected from Ok(127.0.0.1:59058)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802605109Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"ds_connection connected from Ok(127.0.0.1:51165)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802632196Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"ds_connection connected from Ok(127.0.0.1:42137)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.802660914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.803197641Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"1","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.803244347Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"2","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.803349786Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"0","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.803846248Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"2","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.803874715Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"0","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.803890904Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"1","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804452379Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804475807Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"[0]R flush_numbers[0..12]: [129, 129, 130, 130, 119, 124, 129, 128, 129, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804502594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"[0]R generation[0..12]: [70, 70, 71, 71, 65, 67, 70, 69, 70, 71, 71, 71]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804518262Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804532631Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"[1]R flush_numbers[0..12]: [129, 129, 130, 130, 119, 124, 129, 128, 129, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.80454624Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"[1]R generation[0..12]: [70, 70, 71, 71, 65, 67, 70, 69, 70, 71, 71, 71]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804559669Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804573297Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"[2]R flush_numbers[0..12]: [129, 129, 130, 130, 119, 124, 129, 128, 129, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804586656Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"[2]R generation[0..12]: [70, 70, 71, 71, 65, 67, 70, 69, 70, 71, 71, 71]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804603594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804618193Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804630892Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.80464381Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804656399Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"All extents match","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-01-28T17:11:57.804675617Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"":"downstairs","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804688846Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804700635Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"0","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804713774Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"1","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804726372Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"client":"2","":"downstairs","session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804739331Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.8047513Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"a47516c5-d60d-4c88-9dca-5998bab1019b is now active with session: 9882b20c-fadd-4569-9807-2159d777de15","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804763859Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804775617Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475,"session_id":"9882b20c-fadd-4569-9807-2159d777de15"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:11:57.804787616Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 174 len 8192 data: 10 11 02/30 Read block 442 len 40960 03/30 Write block 592 len 28672 data: 5 5 5 4 4 4 4 04/30 Read block 166 len 4096 05/30 Read block 485 len 4096 06/30 Flush 07/30 Flush 08/30 Write block 445 len 28672 data: 6 4 6 5 4 4 4 09/30 Read block 55 len 40960 10/30 Write block 27 len 36864 data: 7 7 7 7 6 6 6 8 7 11/30 Write block 570 len 40960 data: 9 9 9 9 8 6 6 8 10 8 12/30 Read block 52 len 40960 13/30 Write block 384 len 36864 data: 8 6 7 7 8 7 6 7 8 14/30 Write block 164 len 40960 data: 11 8 6 7 9 9 8 8 10 10 15/30 Write block 442 len 36864 data: 7 7 6 7 5 7 6 5 5 16/30 Write block 51 len 20480 data: 7 9 7 7 7 17/30 Write block 68 len 24576 data: 8 7 5 7 7 6 18/30 Write block 260 len 16384 data: 6 8 9 9 19/30 Read block 335 len 20480 20/30 Read block 555 len 4096 21/30 Write block 23 len 4096 data: 6 22/30 Write block 270 len 36864 data: 6 7 5 5 7 8 8 7 8 23/30 Write block 164 len 16384 data: 12 9 7 8 24/30 Flush 25/30 Write block 548 len 8192 data: 4 6 26/30 Write block 167 len 36864 data: 9 10 10 9 9 11 11 11 12 27/30 Read block 101 len 40960 28/30 Read block 580 len 36864 29/30 Write block 441 len 32768 data: 6 8 8 7 8 6 8 7 30/30 Read block 64 len 36864 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 4 6 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:11:57.823919584Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1475} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 129 129 129 F F F 1 020-039 72 72 72 133 133 133 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 72 72 72 133 133 133 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 67 67 67 124 124 124 F F F 6 120-139 70 70 70 129 129 129 F F F 7 140-159 69 69 69 128 128 128 F F F 8 160-179 72 72 72 133 133 133 T T T 9 180-199 71 71 71 130 130 130 F F F 10 200-219 71 71 71 130 130 130 F F F 11 220-239 71 71 71 130 130 130 F F F 12 240-259 69 69 69 128 128 128 F F F 13 260-279 72 72 72 133 133 133 F F F 14 280-299 70 70 70 129 129 129 F F F 15 300-319 70 70 70 129 129 129 F F F 16 320-339 71 71 71 130 130 130 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 71 71 71 130 130 130 F F F 19 380-399 72 72 72 133 133 133 F F F 20 400-419 66 66 66 121 121 121 F F F 21 420-439 70 70 70 129 129 129 F F F 22 440-459 72 72 72 133 133 133 T T T 23 460-479 67 67 67 124 124 124 F F F 24 480-499 70 70 70 129 129 129 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 70 70 70 129 129 129 T T T 28 560-579 72 72 72 133 133 133 F F F 29 580-599 72 72 72 131 131 131 F F F Max gen: 72, Max flush: 133 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 129 129 129 F F F 1 020-039 72 72 72 133 133 133 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 72 72 72 133 133 133 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 67 67 67 124 124 124 F F F 6 120-139 70 70 70 129 129 129 F F F 7 140-159 69 69 69 128 128 128 F F F 8 160-179 73 73 73 134 134 134 F F F 9 180-199 71 71 71 130 130 130 F F F 10 200-219 71 71 71 130 130 130 F F F 11 220-239 71 71 71 130 130 130 F F F 12 240-259 69 69 69 128 128 128 F F F 13 260-279 72 72 72 133 133 133 F F F 14 280-299 70 70 70 129 129 129 F F F 15 300-319 70 70 70 129 129 129 F F F 16 320-339 71 71 71 130 130 130 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 71 71 71 130 130 130 F F F 19 380-399 72 72 72 133 133 133 F F F 20 400-419 66 66 66 121 121 121 F F F 21 420-439 70 70 70 129 129 129 F F F 22 440-459 73 73 73 134 134 134 F F F 23 460-479 67 67 67 124 124 124 F F F 24 480-499 70 70 70 129 129 129 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 73 73 73 134 134 134 F F F 28 560-579 72 72 72 133 133 133 F F F 29 580-599 72 72 72 131 131 131 F F F Max gen: 73, Max flush: 134 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.074806884Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.118115277Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.118825778Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.118867003Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.118881802Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.118896231Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483} {"msg":"Crucible 711538be-a3b3-4b1d-9dd5-b21167763086 has session id: a760ca73-ac1b-492b-9e9d-a1dffbf4eb07","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.119496012Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Upstairs opts: Upstairs UUID: 711538be-a3b3-4b1d-9dd5-b21167763086, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.119542428Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Crucible stats registered with UUID: 711538be-a3b3-4b1d-9dd5-b21167763086","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.119562276Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:02.119575214Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.119994034Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.120027441Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483} {"msg":"711538be-a3b3-4b1d-9dd5-b21167763086 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.120080655Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.120107923Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"0","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.12013113Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"0","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.120148169Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"1","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.120162957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"1","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.120187085Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"2","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.120214962Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"2","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.12023949Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.120266087Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.120288875Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"ds_connection connected from Ok(127.0.0.1:37385)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.120311483Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"ds_connection connected from Ok(127.0.0.1:47473)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.12033689Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"ds_connection connected from Ok(127.0.0.1:47683)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.120363898Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.121020484Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"0","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.12105858Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"1","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.121592318Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"1","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.121718796Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"0","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.122162442Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.122264663Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.122355674Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"2","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.122820229Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"2","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"[0]R flush_numbers[0..12]: [129, 133, 133, 133, 119, 124, 129, 128, 134, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123278784Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"[0]R generation[0..12]: [70, 72, 72, 72, 65, 67, 70, 69, 73, 71, 71, 71]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123302482Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.1233216Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"[1]R flush_numbers[0..12]: [129, 133, 133, 133, 119, 124, 129, 128, 134, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123336058Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"[1]R generation[0..12]: [70, 72, 72, 72, 65, 67, 70, 69, 73, 71, 71, 71]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123349867Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123363825Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"[2]R flush_numbers[0..12]: [129, 133, 133, 133, 119, 124, 129, 128, 134, 130, 130, 130]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123377104Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"[2]R generation[0..12]: [70, 72, 72, 72, 65, 67, 70, 69, 73, 71, 71, 71]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123390263Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123404192Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Max found gen is 74","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.12342325Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123436318Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123448977Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123461686Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"":"downstairs","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123474345Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123485944Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"0","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123498783Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"1","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123514041Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"client":"2","":"downstairs","session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.1235273Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123539299Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"711538be-a3b3-4b1d-9dd5-b21167763086 is now active with session: a760ca73-ac1b-492b-9e9d-a1dffbf4eb07","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123551887Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123563666Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483,"session_id":"a760ca73-ac1b-492b-9e9d-a1dffbf4eb07"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:02.123575625Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 479 len 20480 data: 8 8 7 9 8 02/30 Write block 9 len 28672 data: 7 9 9 8 9 8 9 03/30 Read block 109 len 36864 04/30 Write block 123 len 20480 data: 4 3 3 4 3 05/30 Read block 368 len 24576 06/30 Write block 74 len 4096 data: 6 07/30 Flush 08/30 Write block 310 len 4096 data: 11 09/30 Read block 105 len 16384 10/30 Read block 586 len 24576 11/30 Write block 231 len 4096 data: 4 12/30 Write block 456 len 40960 data: 5 5 4 4 6 5 7 6 7 7 13/30 Read block 215 len 16384 14/30 Write block 549 len 28672 data: 7 8 8 9 9 12 11 15/30 Write block 408 len 28672 data: 7 6 6 5 6 6 6 16/30 Read block 146 len 32768 17/30 Write block 22 len 32768 data: 7 7 6 7 8 8 8 8 18/30 Write block 149 len 40960 data: 4 5 4 3 2 3 4 4 5 8 19/30 Read block 413 len 20480 20/30 Read block 543 len 32768 21/30 Read block 181 len 12288 22/30 Read block 439 len 4096 23/30 Write block 440 len 28672 data: 8 7 9 9 8 9 7 24/30 Read block 51 len 24576 25/30 Read block 270 len 4096 26/30 Read block 427 len 4096 27/30 Read block 474 len 8192 28/30 Write block 374 len 28672 data: 2 3 4 5 7 8 7 29/30 Read block 102 len 12288 30/30 Read block 217 len 16384 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 1 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 22 23 22 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:02.142863507Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1483} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 135 135 135 F F F 1 020-039 72 72 72 133 133 133 T T T 2 040-059 72 72 72 133 133 133 F F F 3 060-079 74 74 74 135 135 135 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 67 67 67 124 124 124 F F F 6 120-139 74 74 74 135 135 135 F F F 7 140-159 69 69 69 128 128 128 T T T 8 160-179 73 73 73 134 134 134 F F F 9 180-199 71 71 71 130 130 130 F F F 10 200-219 71 71 71 130 130 130 F F F 11 220-239 71 71 71 130 130 130 T T T 12 240-259 69 69 69 128 128 128 F F F 13 260-279 72 72 72 133 133 133 F F F 14 280-299 70 70 70 129 129 129 F F F 15 300-319 70 70 70 129 129 129 T T T 16 320-339 71 71 71 130 130 130 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 71 71 71 130 130 130 T T T 19 380-399 72 72 72 133 133 133 T T T 20 400-419 66 66 66 121 121 121 T T T 21 420-439 70 70 70 129 129 129 F F F 22 440-459 73 73 73 134 134 134 T T T 23 460-479 74 74 74 135 135 135 T T T 24 480-499 74 74 74 135 135 135 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 73 73 73 134 134 134 T T T 28 560-579 72 72 72 133 133 133 F F F 29 580-599 72 72 72 131 131 131 F F F Max gen: 74, Max flush: 135 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 135 135 135 F F F 1 020-039 75 75 75 136 136 136 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 74 74 74 135 135 135 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 67 67 67 124 124 124 F F F 6 120-139 74 74 74 135 135 135 F F F 7 140-159 75 75 75 136 136 136 F F F 8 160-179 73 73 73 134 134 134 F F F 9 180-199 71 71 71 130 130 130 F F F 10 200-219 71 71 71 130 130 130 F F F 11 220-239 75 75 75 136 136 136 F F F 12 240-259 69 69 69 128 128 128 F F F 13 260-279 72 72 72 133 133 133 F F F 14 280-299 70 70 70 129 129 129 F F F 15 300-319 75 75 75 136 136 136 F F F 16 320-339 71 71 71 130 130 130 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 75 75 75 136 136 136 F F F 20 400-419 75 75 75 136 136 136 F F F 21 420-439 70 70 70 129 129 129 F F F 22 440-459 75 75 75 136 136 136 F F F 23 460-479 75 75 75 136 136 136 F F F 24 480-499 74 74 74 135 135 135 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 75 75 75 136 136 136 F F F 28 560-579 72 72 72 133 133 133 F F F 29 580-599 72 72 72 131 131 131 F F F Max gen: 75, Max flush: 136 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.402478187Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.445559272Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.446199469Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.446238706Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.446253284Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.446267583Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491} {"msg":"Crucible e118c1cb-32c3-4bce-85ac-ed73b05a7b01 has session id: c0a59939-47ed-49a5-860f-d3b6f78131f1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.446883803Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Upstairs opts: Upstairs UUID: e118c1cb-32c3-4bce-85ac-ed73b05a7b01, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.446921149Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Crucible stats registered with UUID: e118c1cb-32c3-4bce-85ac-ed73b05a7b01","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.446937118Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:06.446950026Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447383984Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447408912Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491} {"msg":"e118c1cb-32c3-4bce-85ac-ed73b05a7b01 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447476845Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447495513Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"0","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447517321Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"0","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.44753073Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"1","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447547138Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"1","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447560547Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"2","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447583225Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"2","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447605862Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.44762991Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447654898Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:56489)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447776296Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45808)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447816172Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:39070)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.447844729Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.448542591Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"2","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.448574958Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"0","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.449178399Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"0","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.449215336Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"1","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.449243583Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"2","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.449740085Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.449835425Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"1","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.449861273Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"[0]R flush_numbers[0..12]: [135, 136, 133, 135, 119, 124, 135, 136, 134, 130, 130, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450370883Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"[0]R generation[0..12]: [74, 75, 72, 74, 65, 67, 74, 75, 73, 71, 71, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450398121Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450413069Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"[1]R flush_numbers[0..12]: [135, 136, 133, 135, 119, 124, 135, 136, 134, 130, 130, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450427288Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"[1]R generation[0..12]: [74, 75, 72, 74, 65, 67, 74, 75, 73, 71, 71, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450440926Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450454605Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"[2]R flush_numbers[0..12]: [135, 136, 133, 135, 119, 124, 135, 136, 134, 130, 130, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450468054Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"[2]R generation[0..12]: [74, 75, 72, 74, 65, 67, 74, 75, 73, 71, 71, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450481402Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450498961Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:12:06.450517709Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450550516Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450576353Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450593222Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"":"downstairs","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.45060632Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450625388Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"0","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450650266Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"1","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450673004Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"client":"2","":"downstairs","session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450697761Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450721599Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"e118c1cb-32c3-4bce-85ac-ed73b05a7b01 is now active with session: c0a59939-47ed-49a5-860f-d3b6f78131f1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450743567Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450756316Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491,"session_id":"c0a59939-47ed-49a5-860f-d3b6f78131f1"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:06.450769854Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 132 len 4096 data: 6 02/30 Read block 379 len 36864 03/30 Write block 449 len 32768 data: 6 6 5 4 5 5 6 6 04/30 Read block 256 len 12288 05/30 Write block 276 len 28672 data: 9 8 9 8 6 5 5 06/30 Flush 07/30 Write block 113 len 36864 data: 7 8 7 7 7 5 5 5 4 08/30 Write block 218 len 36864 data: 4 4 5 5 5 7 7 8 9 09/30 Read block 246 len 4096 10/30 Read block 294 len 16384 11/30 Write block 447 len 36864 data: 9 8 7 7 6 5 6 6 7 12/30 Read block 83 len 16384 13/30 Write block 431 len 40960 data: 5 4 8 9 8 9 9 9 8 9 14/30 Read block 518 len 12288 15/30 Flush 16/30 Write block 296 len 40960 data: 5 5 5 5 7 7 5 5 4 4 17/30 Flush 18/30 Write block 464 len 40960 data: 8 8 7 6 4 5 5 5 4 4 19/30 Write block 69 len 32768 data: 8 6 8 8 7 7 6 6 20/30 Flush 21/30 Write block 426 len 16384 data: 5 5 4 4 22/30 Read block 309 len 24576 23/30 Write block 265 len 20480 data: 11 11 10 8 7 24/30 Write block 9 len 16384 data: 8 10 10 9 25/30 Read block 414 len 28672 26/30 Read block 199 len 12288 27/30 Write block 303 len 32768 data: 6 5 5 6 7 8 9 12 28/30 Write block 248 len 4096 data: 10 29/30 Write block 69 len 4096 data: 9 30/30 Read block 209 len 4096 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:06.471416234Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1491} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 135 135 135 T T T 1 020-039 75 75 75 136 136 136 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 76 76 76 140 140 140 T T T 4 080-099 65 65 65 119 119 119 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 76 76 76 138 138 138 F F F 7 140-159 75 75 75 136 136 136 F F F 8 160-179 73 73 73 134 134 134 F F F 9 180-199 71 71 71 130 130 130 F F F 10 200-219 76 76 76 138 138 138 F F F 11 220-239 76 76 76 138 138 138 F F F 12 240-259 69 69 69 128 128 128 T T T 13 260-279 76 76 76 137 137 137 T T T 14 280-299 76 76 76 139 139 139 F F F 15 300-319 76 76 76 139 139 139 T T T 16 320-339 71 71 71 130 130 130 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 75 75 75 136 136 136 F F F 20 400-419 75 75 75 136 136 136 F F F 21 420-439 76 76 76 138 138 138 T T T 22 440-459 76 76 76 138 138 138 F F F 23 460-479 76 76 76 140 140 140 F F F 24 480-499 74 74 74 135 135 135 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 75 75 75 136 136 136 F F F 28 560-579 72 72 72 133 133 133 F F F 29 580-599 72 72 72 131 131 131 F F F Max gen: 76, Max flush: 140 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 75 75 75 136 136 136 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 77 77 77 141 141 141 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 76 76 76 138 138 138 F F F 7 140-159 75 75 75 136 136 136 F F F 8 160-179 73 73 73 134 134 134 F F F 9 180-199 71 71 71 130 130 130 F F F 10 200-219 76 76 76 138 138 138 F F F 11 220-239 76 76 76 138 138 138 F F F 12 240-259 77 77 77 141 141 141 F F F 13 260-279 77 77 77 141 141 141 F F F 14 280-299 76 76 76 139 139 139 F F F 15 300-319 77 77 77 141 141 141 F F F 16 320-339 71 71 71 130 130 130 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 75 75 75 136 136 136 F F F 20 400-419 75 75 75 136 136 136 F F F 21 420-439 77 77 77 141 141 141 F F F 22 440-459 76 76 76 138 138 138 F F F 23 460-479 76 76 76 140 140 140 F F F 24 480-499 74 74 74 135 135 135 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 75 75 75 136 136 136 F F F 28 560-579 72 72 72 133 133 133 F F F 29 580-599 72 72 72 131 131 131 F F F Max gen: 77, Max flush: 141 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.706719891Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.749745291Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.750328105Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.75036895Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.750386179Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.750400718Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499} {"msg":"Crucible f190df7d-85f6-49b5-8e41-1816215ff2ae has session id: 46efa35a-ae8e-48d4-8492-2e45ef2efebb","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751099519Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Upstairs opts: Upstairs UUID: f190df7d-85f6-49b5-8e41-1816215ff2ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751147215Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Crucible stats registered with UUID: f190df7d-85f6-49b5-8e41-1816215ff2ae","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751167303Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:10.751180682Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751572433Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.75160541Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499} {"msg":"f190df7d-85f6-49b5-8e41-1816215ff2ae active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751668644Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751694641Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"0","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751716579Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"0","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751730258Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"1","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751744257Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"1","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751757305Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"2","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751771124Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"2","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751784543Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.751798311Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.75181546Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"ds_connection connected from Ok(127.0.0.1:39663)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.752096722Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"ds_connection connected from Ok(127.0.0.1:52381)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.752140618Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"ds_connection connected from Ok(127.0.0.1:34925)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.752218461Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.752662687Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"2","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.752691315Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"0","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.753161189Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"1","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.753335772Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"0","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.753374898Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"2","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.753684428Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"1","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.753912176Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.753935253Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"[0]R flush_numbers[0..12]: [141, 136, 133, 141, 119, 138, 138, 136, 134, 130, 138, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754157532Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"[0]R generation[0..12]: [77, 75, 72, 77, 65, 76, 76, 75, 73, 71, 76, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754179539Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754198248Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"[1]R flush_numbers[0..12]: [141, 136, 133, 141, 119, 138, 138, 136, 134, 130, 138, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754212406Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"[1]R generation[0..12]: [77, 75, 72, 77, 65, 76, 76, 75, 73, 71, 76, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754225855Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754239594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"[2]R flush_numbers[0..12]: [141, 136, 133, 141, 119, 138, 138, 136, 134, 130, 138, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754253142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"[2]R generation[0..12]: [77, 75, 72, 77, 65, 76, 76, 75, 73, 71, 76, 76]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754266441Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.75428008Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754292699Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"Wait for a query_work_queue command to finish before sending IO ":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754312477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Next flush: 142","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754324985Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754337424Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"":"downstairs","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754350433Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754362252Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"0","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754375511Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"1","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754388579Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"client":"2","":"downstairs","session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754403398Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754416407Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"f190df7d-85f6-49b5-8e41-1816215ff2ae is now active with session: 46efa35a-ae8e-48d4-8492-2e45ef2efebb","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754429095Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754440574Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499,"session_id":"46efa35a-ae8e-48d4-8492-2e45ef2efebb"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:10.754452453Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 229 len 4096 data: 8 02/30 Write block 322 len 36864 data: 7 6 7 8 9 10 11 9 8 03/30 Read block 420 len 40960 04/30 Write block 484 len 16384 data: 6 7 6 6 05/30 Read block 196 len 36864 06/30 Write block 147 len 12288 data: 5 4 5 07/30 Write block 540 len 28672 data: 5 4 3 3 3 6 5 08/30 Read block 48 len 28672 09/30 Write block 154 len 4096 data: 4 10/30 Read block 159 len 40960 11/30 Write block 577 len 28672 data: 9 11 9 9 9 11 10 12/30 Write block 404 len 28672 data: 6 7 7 6 8 7 7 13/30 Write block 437 len 20480 data: 10 10 9 10 8 14/30 Write block 487 len 4096 data: 7 15/30 Flush 16/30 Write block 320 len 24576 data: 4 5 8 7 8 9 17/30 Read block 495 len 20480 18/30 Read block 63 len 8192 19/30 Read block 434 len 28672 20/30 Write block 580 len 20480 data: 10 10 12 11 12 21/30 Write block 329 len 16384 data: 10 9 8 7 22/30 Read block 76 len 28672 23/30 Write block 309 len 20480 data: 10 13 11 10 10 24/30 Read block 345 len 32768 25/30 Write block 593 len 24576 data: 6 6 5 5 5 5 26/30 Read block 27 len 12288 27/30 Read block 232 len 4096 28/30 Read block 530 len 8192 29/30 Write block 200 len 32768 data: 5 4 5 7 8 7 7 6 30/30 Read block 311 len 40960 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 15 13 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:10.773199867Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1499} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 75 75 75 136 136 136 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 77 77 77 141 141 141 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 76 76 76 138 138 138 F F F 7 140-159 78 78 78 142 142 142 F F F 8 160-179 73 73 73 134 134 134 F F F 9 180-199 71 71 71 130 130 130 F F F 10 200-219 76 76 76 138 138 138 T T T 11 220-239 78 78 78 142 142 142 F F F 12 240-259 77 77 77 141 141 141 F F F 13 260-279 77 77 77 141 141 141 F F F 14 280-299 76 76 76 139 139 139 F F F 15 300-319 77 77 77 141 141 141 T T T 16 320-339 78 78 78 142 142 142 T T T 17 340-359 71 71 71 130 130 130 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 75 75 75 136 136 136 F F F 20 400-419 78 78 78 142 142 142 F F F 21 420-439 78 78 78 142 142 142 F F F 22 440-459 78 78 78 142 142 142 F F F 23 460-479 76 76 76 140 140 140 F F F 24 480-499 78 78 78 142 142 142 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 78 78 78 142 142 142 F F F 28 560-579 78 78 78 142 142 142 F F F 29 580-599 78 78 78 142 142 142 T T T Max gen: 78, Max flush: 142 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 75 75 75 136 136 136 F F F 2 040-059 72 72 72 133 133 133 F F F 3 060-079 77 77 77 141 141 141 F F F 4 080-099 65 65 65 119 119 119 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 76 76 76 138 138 138 F F F 7 140-159 78 78 78 142 142 142 F F F 8 160-179 73 73 73 134 134 134 F F F 9 180-199 71 71 71 130 130 130 F F F 10 200-219 79 79 79 143 143 143 F F F 11 220-239 78 78 78 142 142 142 F F F 12 240-259 77 77 77 141 141 141 F F F 13 260-279 77 77 77 141 141 141 F F F 14 280-299 76 76 76 139 139 139 F F F 15 300-319 79 79 79 143 143 143 F F F 16 320-339 79 79 79 143 143 143 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 75 75 75 136 136 136 F F F 20 400-419 78 78 78 142 142 142 F F F 21 420-439 78 78 78 142 142 142 F F F 22 440-459 78 78 78 142 142 142 F F F 23 460-479 76 76 76 140 140 140 F F F 24 480-499 78 78 78 142 142 142 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 78 78 78 142 142 142 F F F 28 560-579 78 78 78 142 142 142 F F F 29 580-599 79 79 79 143 143 143 F F F Max gen: 79, Max flush: 143 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.003970585Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.04859469Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.049185653Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.049234668Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.049254226Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.049272194Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507} {"msg":"Crucible 86cd43c6-fe59-4506-b19d-bf24e72cad1b has session id: fafb87a6-3ce0-4e2f-8c57-3668c4064bda","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.049816241Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Upstairs opts: Upstairs UUID: 86cd43c6-fe59-4506-b19d-bf24e72cad1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.049862057Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Crucible stats registered with UUID: 86cd43c6-fe59-4506-b19d-bf24e72cad1b","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.049888894Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:15.049904803Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050316222Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050348519Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507} {"msg":"86cd43c6-fe59-4506-b19d-bf24e72cad1b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050402704Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050432651Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"0","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050474047Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"0","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050504174Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"1","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050532351Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"1","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050559039Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"2","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050583336Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"2","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050605144Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050632052Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050657769Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"ds_connection connected from Ok(127.0.0.1:49625)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050682027Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"ds_connection connected from Ok(127.0.0.1:48520)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050706204Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"ds_connection connected from Ok(127.0.0.1:33315)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.050863389Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.051307886Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"0","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.051336593Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"2","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.052001908Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"2","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.052039305Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"0","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.052595201Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.052709909Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"1","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.052742456Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053243157Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"1","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"[0]R flush_numbers[0..12]: [141, 136, 133, 141, 119, 138, 138, 142, 134, 130, 143, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053720091Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"[0]R generation[0..12]: [77, 75, 72, 77, 65, 76, 76, 78, 73, 71, 79, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053741729Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053756847Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"[1]R flush_numbers[0..12]: [141, 136, 133, 141, 119, 138, 138, 142, 134, 130, 143, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053770906Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"[1]R generation[0..12]: [77, 75, 72, 77, 65, 76, 76, 78, 73, 71, 79, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053784235Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053798023Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"[2]R flush_numbers[0..12]: [141, 136, 133, 141, 119, 138, 138, 142, 134, 130, 143, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053811252Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"[2]R generation[0..12]: [77, 75, 72, 77, 65, 76, 76, 78, 73, 71, 79, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053824481Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053838189Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053854318Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053874636Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053887625Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053900293Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"":"downstairs","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053913332Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053925041Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"0","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.05393835Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"1","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053951329Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"client":"2","":"downstairs","session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053964147Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053976316Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"86cd43c6-fe59-4506-b19d-bf24e72cad1b is now active with session: fafb87a6-3ce0-4e2f-8c57-3668c4064bda","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.053988945Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.054000574Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507,"session_id":"fafb87a6-3ce0-4e2f-8c57-3668c4064bda"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:15.054012763Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 430 len 40960 data: 5 6 5 9 10 9 10 11 11 10 02/30 Write block 51 len 4096 data: 8 03/30 Write block 443 len 16384 data: 10 9 10 8 04/30 Write block 423 len 36864 data: 7 5 5 6 6 5 5 6 7 05/30 Read block 472 len 32768 06/30 Read block 555 len 40960 07/30 Read block 50 len 12288 08/30 Read block 307 len 28672 09/30 Write block 179 len 24576 data: 9 11 9 11 11 9 10/30 Read block 294 len 20480 11/30 Write block 286 len 24576 data: 4 4 4 5 6 6 12/30 Read block 596 len 8192 13/30 Write block 324 len 24576 data: 9 10 10 11 12 11 14/30 Read block 384 len 28672 15/30 Write block 236 len 40960 data: 6 7 9 9 10 10 10 10 8 7 16/30 Write block 38 len 24576 data: 8 7 7 5 5 6 17/30 Read block 355 len 32768 18/30 Read block 586 len 36864 19/30 Write block 210 len 16384 data: 8 6 7 7 20/30 Read block 501 len 36864 21/30 Read block 213 len 4096 22/30 Read block 289 len 24576 23/30 Write block 89 len 40960 data: 7 8 7 6 6 5 5 6 6 5 24/30 Read block 137 len 36864 25/30 Read block 541 len 28672 26/30 Write block 309 len 12288 data: 11 14 12 27/30 Read block 564 len 20480 28/30 Read block 84 len 4096 29/30 Read block 317 len 40960 30/30 Write block 399 len 24576 data: 5 6 6 6 9 7 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:15.073984068Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1507} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 75 75 75 136 136 136 T T T 2 040-059 72 72 72 133 133 133 T T T 3 060-079 77 77 77 141 141 141 F F F 4 080-099 65 65 65 119 119 119 T T T 5 100-119 76 76 76 138 138 138 F F F 6 120-139 76 76 76 138 138 138 F F F 7 140-159 78 78 78 142 142 142 F F F 8 160-179 73 73 73 134 134 134 T T T 9 180-199 71 71 71 130 130 130 T T T 10 200-219 79 79 79 143 143 143 T T T 11 220-239 78 78 78 142 142 142 T T T 12 240-259 77 77 77 141 141 141 T T T 13 260-279 77 77 77 141 141 141 F F F 14 280-299 76 76 76 139 139 139 T T T 15 300-319 79 79 79 143 143 143 T T T 16 320-339 79 79 79 143 143 143 T T T 17 340-359 71 71 71 130 130 130 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 75 75 75 136 136 136 T T T 20 400-419 78 78 78 142 142 142 T T T 21 420-439 78 78 78 142 142 142 T T T 22 440-459 78 78 78 142 142 142 T T T 23 460-479 76 76 76 140 140 140 F F F 24 480-499 78 78 78 142 142 142 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 78 78 78 142 142 142 F F F 28 560-579 78 78 78 142 142 142 F F F 29 580-599 79 79 79 143 143 143 F F F Max gen: 79, Max flush: 143 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 80 80 80 144 144 144 F F F 3 060-079 77 77 77 141 141 141 F F F 4 080-099 80 80 80 144 144 144 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 76 76 76 138 138 138 F F F 7 140-159 78 78 78 142 142 142 F F F 8 160-179 80 80 80 144 144 144 F F F 9 180-199 80 80 80 144 144 144 F F F 10 200-219 80 80 80 144 144 144 F F F 11 220-239 80 80 80 144 144 144 F F F 12 240-259 80 80 80 144 144 144 F F F 13 260-279 77 77 77 141 141 141 F F F 14 280-299 80 80 80 144 144 144 F F F 15 300-319 80 80 80 144 144 144 F F F 16 320-339 80 80 80 144 144 144 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 80 80 80 144 144 144 F F F 20 400-419 80 80 80 144 144 144 F F F 21 420-439 80 80 80 144 144 144 F F F 22 440-459 80 80 80 144 144 144 F F F 23 460-479 76 76 76 140 140 140 F F F 24 480-499 78 78 78 142 142 142 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 78 78 78 142 142 142 F F F 28 560-579 78 78 78 142 142 142 F F F 29 580-599 79 79 79 143 143 143 F F F Max gen: 80, Max flush: 144 Begin loop 41 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.355379869Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:29.396716364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.397095517Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.427864821Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.428415547Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.428457233Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.428475551Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.42849031Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515} {"msg":"Crucible 1a9f6d75-09e7-459c-8a5f-c37fcc2bc0bb has session id: 9ef7af49-a12b-407a-811f-5f93794e7f1c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429036907Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Upstairs opts: Upstairs UUID: 1a9f6d75-09e7-459c-8a5f-c37fcc2bc0bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429081702Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Crucible stats registered with UUID: 1a9f6d75-09e7-459c-8a5f-c37fcc2bc0bb","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.42910427Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:29.429118069Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429562665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429585603Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515} {"msg":"1a9f6d75-09e7-459c-8a5f-c37fcc2bc0bb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429648397Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429665665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"0","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429687853Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"0","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429702702Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"1","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.42971713Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"1","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429734479Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"2","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429748877Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"2","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429767575Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429792323Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429817131Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57866)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.429843868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62693)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.42992819Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49850)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.430100003Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.430496954Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"1","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.430614063Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"2","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.430811214Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"0","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.431085857Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"1","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.431239452Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"2","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.431520275Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"0","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.431710066Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.431797188Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"[0]R flush_numbers[0..12]: [141, 144, 144, 141, 144, 138, 138, 142, 144, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.431996548Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"[0]R generation[0..12]: [77, 80, 80, 77, 80, 76, 76, 78, 80, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432017346Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432032215Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"[1]R flush_numbers[0..12]: [141, 144, 144, 141, 144, 138, 138, 142, 144, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432045774Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"[1]R generation[0..12]: [77, 80, 80, 77, 80, 76, 76, 78, 80, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432058992Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432072831Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"[2]R flush_numbers[0..12]: [141, 144, 144, 141, 144, 138, 138, 142, 144, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.43208647Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"[2]R generation[0..12]: [77, 80, 80, 77, 80, 76, 76, 78, 80, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432099878Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432113307Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Max found gen is 81","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432126056Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Generation requested: 82 >= found:81","vWait for a query_work_queue command to finish before sending IO ":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432145474Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432158413Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432170961Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"":"downstairs","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.43218802Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432200108Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"0","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432213117Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"1","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432225876Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"client":"2","":"downstairs","session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432238835Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432250734Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"1a9f6d75-09e7-459c-8a5f-c37fcc2bc0bb is now active with session: 9ef7af49-a12b-407a-811f-5f93794e7f1c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432263262Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.432274941Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515,"session_id":"9ef7af49-a12b-407a-811f-5f93794e7f1c"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:29.43228678Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 450 len 4096 data: 8 02/30 Write block 172 len 36864 data: 12 12 12 13 10 9 8 10 12 03/30 Read block 332 len 20480 04/30 Read block 541 len 40960 05/30 Write block 548 len 24576 data: 5 8 9 9 10 10 06/30 Read block 307 len 40960 07/30 Read block 491 len 20480 08/30 Read block 398 len 28672 09/30 Write block 420 len 20480 data: 4 5 7 8 6 10/30 Write block 199 len 28672 data: 5 6 5 6 8 9 8 11/30 Read block 362 len 20480 12/30 Flush 13/30 Write block 493 len 28672 data: 9 9 9 7 5 6 6 14/30 Flush 15/30 Write block 560 len 12288 data: 5 4 5 16/30 Read block 354 len 20480 17/30 Write block 216 len 36864 data: 6 5 5 5 6 6 6 8 8 18/30 Read block 59 len 16384 19/30 Read block 540 len 32768 20/30 Read block 83 len 40960 21/30 Read block 170 len 24576 22/30 Read block 514 len 28672 23/30 Write block 569 len 12288 data: 8 10 10 24/30 Read block 285 len 4096 25/30 Read block 563 len 20480 26/30 Write block 589 len 24576 data: 7 5 4 6 7 7 27/30 Write block 140 len 24576 data: 16 14 11 10 8 7 28/30 Read block 291 len 40960 29/30 Read block 516 len 32768 30/30 Read block 280 len 20480 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 3 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 16 16 15 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:29.453402824Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1515} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 80 80 80 144 144 144 F F F 3 060-079 77 77 77 141 141 141 F F F 4 080-099 80 80 80 144 144 144 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 76 76 76 138 138 138 F F F 7 140-159 78 78 78 142 142 142 T T T 8 160-179 82 82 82 145 145 145 F F F 9 180-199 82 82 82 145 145 145 F F F 10 200-219 82 82 82 145 145 145 T T T 11 220-239 80 80 80 144 144 144 T T T 12 240-259 80 80 80 144 144 144 F F F 13 260-279 77 77 77 141 141 141 F F F 14 280-299 80 80 80 144 144 144 F F F 15 300-319 80 80 80 144 144 144 F F F 16 320-339 80 80 80 144 144 144 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 80 80 80 144 144 144 F F F 20 400-419 80 80 80 144 144 144 F F F 21 420-439 82 82 82 145 145 145 F F F 22 440-459 82 82 82 145 145 145 F F F 23 460-479 76 76 76 140 140 140 F F F 24 480-499 82 82 82 146 146 146 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 82 82 82 145 145 145 F F F 28 560-579 78 78 78 142 142 142 T T T 29 580-599 79 79 79 143 143 143 T T T Max gen: 82, Max flush: 146 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 80 80 80 144 144 144 F F F 3 060-079 77 77 77 141 141 141 F F F 4 080-099 80 80 80 144 144 144 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 76 76 76 138 138 138 F F F 7 140-159 83 83 83 147 147 147 F F F 8 160-179 82 82 82 145 145 145 F F F 9 180-199 82 82 82 145 145 145 F F F 10 200-219 83 83 83 147 147 147 F F F 11 220-239 83 83 83 147 147 147 F F F 12 240-259 80 80 80 144 144 144 F F F 13 260-279 77 77 77 141 141 141 F F F 14 280-299 80 80 80 144 144 144 F F F 15 300-319 80 80 80 144 144 144 F F F 16 320-339 80 80 80 144 144 144 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 80 80 80 144 144 144 F F F 20 400-419 80 80 80 144 144 144 F F F 21 420-439 82 82 82 145 145 145 F F F 22 440-459 82 82 82 145 145 145 F F F 23 460-479 76 76 76 140 140 140 F F F 24 480-499 82 82 82 146 146 146 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 82 82 82 145 145 145 F F F 28 560-579 83 83 83 147 147 147 F F F 29 580-599 83 83 83 147 147 147 F F F Max gen: 83, Max flush: 147 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.712695464Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:33.754354258Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.754731841Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.785504594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.786134812Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.786169709Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.786183898Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.786198076Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523} {"msg":"Crucible 8f58f87c-f15c-46c2-89ae-8dd5654942f6 has session id: e4f935d7-6e62-43d5-8c0d-bd311a5d1e02","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.786802347Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Upstairs opts: Upstairs UUID: 8f58f87c-f15c-46c2-89ae-8dd5654942f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.786848853Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Crucible stats registered with UUID: 8f58f87c-f15c-46c2-89ae-8dd5654942f6","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.786870591Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:33.786884699Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787316957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787340045Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523} {"msg":"8f58f87c-f15c-46c2-89ae-8dd5654942f6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787400859Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787418838Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"0","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787441865Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"0","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787466443Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"1","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.78749278Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"1","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787523597Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"2","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787550535Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"2","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787565193Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787578852Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.78759219Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"ds_connection connected from Ok(127.0.0.1:48231)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787620398Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"ds_connection connected from Ok(127.0.0.1:47968)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787648075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"ds_connection connected from Ok(127.0.0.1:34932)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.787794401Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.788437418Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"0","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.788464886Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"2","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.788489403Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"1","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.789125751Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"1","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.789159258Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"2","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.789187485Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"0","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.789736132Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.789843142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"[0]R flush_numbers[0..12]: [141, 144, 144, 141, 144, 138, 138, 147, 145, 145, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.789881818Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"[0]R generation[0..12]: [77, 80, 80, 77, 80, 76, 76, 83, 82, 82, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.789898816Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.789913215Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"[1]R flush_numbers[0..12]: [141, 144, 144, 141, 144, 138, 138, 147, 145, 145, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.789926723Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"[1]R generation[0..12]: [77, 80, 80, 77, 80, 76, 76, 83, 82, 82, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.789940322Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.78995867Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"[2]R flush_numbers[0..12]: [141, 144, 144, 141, 144, 138, 138, 147, 145, 145, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.789972999Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"[2]R generation[0..12]: [77, 80, 80, 77, 80, 76, 76, 83, 82, 82, 83, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.789986377Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790000046Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790012615Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790025374Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Next flush: 148","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790037932Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790050521Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"":"downstairs","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790069189Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790088198Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"0","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790112725Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"1","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790127094Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"client":"2","":"downstairs","session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790139962Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790151751Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"8f58f87c-f15c-46c2-89ae-8dd5654942f6 is now active with session: e4f935d7-6e62-43d5-8c0d-bd311a5d1e02","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.79016791Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790192717Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523,"session_id":"e4f935d7-6e62-43d5-8c0d-bd311a5d1e02"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:33.790208116Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 260 len 36864 data: 7 9 10 10 9 12 12 11 9 02/30 Read block 321 len 20480 03/30 Read block 383 len 12288 04/30 Write block 67 len 24576 data: 9 9 10 7 9 9 05/30 Flush 06/30 Write block 192 len 8192 data: 7 7 07/30 Read block 24 len 36864 08/30 Write block 247 len 4096 data: 8 09/30 Write block 582 len 40960 data: 13 12 13 12 8 6 5 8 6 5 10/30 Read block 251 len 36864 11/30 Read block 458 len 16384 12/30 Flush 13/30 Write block 237 len 24576 data: 8 10 10 11 11 11 14/30 Flush 15/30 Write block 569 len 28672 data: 9 11 11 10 10 9 7 16/30 Write block 428 len 12288 data: 6 6 7 17/30 Flush 18/30 Write block 270 len 36864 data: 7 8 6 6 8 9 10 9 10 19/30 Read block 398 len 16384 20/30 Write block 553 len 40960 data: 11 13 12 10 9 8 7 6 5 6 21/30 Read block 64 len 32768 22/30 Read block 459 len 4096 23/30 Write block 403 len 24576 data: 10 8 8 8 7 9 24/30 Write block 213 len 8192 data: 8 7 25/30 Write block 396 len 4096 data: 4 26/30 Read block 550 len 24576 27/30 Write block 287 len 28672 data: 5 5 6 7 7 7 6 28/30 Read block 331 len 24576 29/30 Write block 465 len 4096 data: 9 30/30 Read block 205 len 4096 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 9 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:33.810080891Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1523} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 80 80 80 144 144 144 F F F 3 060-079 84 84 84 148 148 148 F F F 4 080-099 80 80 80 144 144 144 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 76 76 76 138 138 138 F F F 7 140-159 83 83 83 147 147 147 F F F 8 160-179 82 82 82 145 145 145 F F F 9 180-199 84 84 84 149 149 149 F F F 10 200-219 83 83 83 147 147 147 T T T 11 220-239 84 84 84 150 150 150 F F F 12 240-259 84 84 84 150 150 150 F F F 13 260-279 84 84 84 148 148 148 T T T 14 280-299 80 80 80 144 144 144 T T T 15 300-319 80 80 80 144 144 144 F F F 16 320-339 80 80 80 144 144 144 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 80 80 80 144 144 144 T T T 20 400-419 80 80 80 144 144 144 T T T 21 420-439 84 84 84 151 151 151 F F F 22 440-459 82 82 82 145 145 145 F F F 23 460-479 76 76 76 140 140 140 T T T 24 480-499 82 82 82 146 146 146 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 82 82 82 145 145 145 T T T 28 560-579 84 84 84 151 151 151 T T T 29 580-599 84 84 84 149 149 149 F F F Max gen: 84, Max flush: 151 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 80 80 80 144 144 144 F F F 3 060-079 84 84 84 148 148 148 F F F 4 080-099 80 80 80 144 144 144 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 76 76 76 138 138 138 F F F 7 140-159 83 83 83 147 147 147 F F F 8 160-179 82 82 82 145 145 145 F F F 9 180-199 84 84 84 149 149 149 F F F 10 200-219 85 85 85 152 152 152 F F F 11 220-239 84 84 84 150 150 150 F F F 12 240-259 84 84 84 150 150 150 F F F 13 260-279 85 85 85 152 152 152 F F F 14 280-299 85 85 85 152 152 152 F F F 15 300-319 80 80 80 144 144 144 F F F 16 320-339 80 80 80 144 144 144 F F F 17 340-359 71 71 71 130 130 130 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 85 85 85 152 152 152 F F F 20 400-419 85 85 85 152 152 152 F F F 21 420-439 84 84 84 151 151 151 F F F 22 440-459 82 82 82 145 145 145 F F F 23 460-479 85 85 85 152 152 152 F F F 24 480-499 82 82 82 146 146 146 F F F 25 500-519 65 65 65 119 119 119 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 85 85 85 152 152 152 F F F 28 560-579 85 85 85 152 152 152 F F F 29 580-599 84 84 84 149 149 149 F F F Max gen: 85, Max flush: 152 Begin loop 43 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.090240991Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.133877702Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.134467584Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.134508831Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.134527838Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.134542247Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531} {"msg":"Crucible b684c2cc-15b2-4bd6-9882-2347b03a01c2 has session id: b6a729ed-778b-434c-9416-3f81aa538d95","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135009172Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Upstairs opts: Upstairs UUID: b684c2cc-15b2-4bd6-9882-2347b03a01c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135054717Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Crucible stats registered with UUID: b684c2cc-15b2-4bd6-9882-2347b03a01c2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135081705Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:38.135103833Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.13563884Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135676957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531} {"msg":"b684c2cc-15b2-4bd6-9882-2347b03a01c2 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135721472Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.13574498Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"0","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135775667Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"0","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135800205Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"1","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135825402Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"1","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.1358519Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"2","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135878427Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"2","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135903815Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135931862Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135958879Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"ds_connection connected from Ok(127.0.0.1:42593)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.135984957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"ds_connection connected from Ok(127.0.0.1:37146)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.136010504Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"ds_connection connected from Ok(127.0.0.1:41416)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.136240292Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.136735354Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"0","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.136764931Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"1","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.136780029Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"2","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.137351963Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"0","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.137475182Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"1","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.137603599Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"2","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.137941616Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138098561Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"[0]R flush_numbers[0..12]: [141, 144, 144, 148, 144, 138, 138, 147, 145, 149, 152, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138214539Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"[0]R generation[0..12]: [77, 80, 80, 84, 80, 76, 76, 83, 82, 84, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138234928Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138249496Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"[1]R flush_numbers[0..12]: [141, 144, 144, 148, 144, 138, 138, 147, 145, 149, 152, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138263215Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"[1]R generation[0..12]: [77, 80, 80, 84, 80, 76, 76, 83, 82, 84, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138276893Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138290622Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"[2]R flush_numbers[0..12]: [141, 144, 144, 148, 144, 138, 138, 147, 145, 149, 152, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138304861Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"[2]R generation[0..12]: [77, 80, 80, 84, 80, 76, 76, 83, 82, 84, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138318269Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138331928Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138344357Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138363785Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138376574Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138389293Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"":"downstairs","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138406781Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.13841904Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"0","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138432118Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"1","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138445027Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"client":"2","":"downstairs","session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138457946Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138469775Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"b684c2cc-15b2-4bd6-9882-2347b03a01c2 is now active with session: b6a729ed-778b-434c-9416-3f81aa538d95","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138482313Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138498332Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531,"session_id":"b6a729ed-778b-434c-9416-3f81aa538d95"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:38.138510431Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 321 len 16384 data: 6 9 8 10 02/30 Flush 03/30 Write block 289 len 36864 data: 7 8 8 8 7 6 4 6 6 04/30 Write block 133 len 8192 data: 6 5 05/30 Read block 404 len 32768 06/30 Flush 07/30 Write block 445 len 12288 data: 11 9 10 08/30 Write block 395 len 4096 data: 4 09/30 Write block 496 len 28672 data: 8 6 7 7 6 5 4 10/30 Write block 142 len 24576 data: 12 11 9 8 7 6 11/30 Write block 253 len 28672 data: 9 8 6 9 7 8 6 12/30 Write block 446 len 32768 data: 10 11 9 8 9 7 6 7 13/30 Write block 186 len 28672 data: 8 8 6 7 7 7 8 14/30 Read block 525 len 24576 15/30 Read block 57 len 8192 16/30 Read block 90 len 20480 17/30 Write block 52 len 12288 data: 10 8 8 18/30 Read block 256 len 4096 19/30 Read block 224 len 24576 20/30 Read block 397 len 24576 21/30 Read block 11 len 36864 22/30 Read block 351 len 12288 23/30 Write block 161 len 40960 data: 8 9 10 13 10 8 10 11 11 10 24/30 Read block 36 len 24576 25/30 Read block 123 len 24576 26/30 Write block 354 len 20480 data: 6 5 4 5 8 27/30 Write block 577 len 8192 data: 10 12 28/30 Read block 136 len 20480 29/30 Read block 103 len 4096 30/30 Write block 262 len 8192 data: 11 11 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 3 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:38.156299549Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1531} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 80 80 80 144 144 144 T T T 3 060-079 84 84 84 148 148 148 F F F 4 080-099 80 80 80 144 144 144 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 86 86 86 154 154 154 F F F 7 140-159 83 83 83 147 147 147 T T T 8 160-179 82 82 82 145 145 145 T T T 9 180-199 84 84 84 149 149 149 T T T 10 200-219 85 85 85 152 152 152 F F F 11 220-239 84 84 84 150 150 150 F F F 12 240-259 84 84 84 150 150 150 T T T 13 260-279 85 85 85 152 152 152 T T T 14 280-299 86 86 86 154 154 154 F F F 15 300-319 80 80 80 144 144 144 F F F 16 320-339 86 86 86 153 153 153 F F F 17 340-359 71 71 71 130 130 130 T T T 18 360-379 75 75 75 136 136 136 F F F 19 380-399 85 85 85 152 152 152 T T T 20 400-419 85 85 85 152 152 152 F F F 21 420-439 84 84 84 151 151 151 F F F 22 440-459 82 82 82 145 145 145 T T T 23 460-479 85 85 85 152 152 152 F F F 24 480-499 82 82 82 146 146 146 T T T 25 500-519 65 65 65 119 119 119 T T T 26 520-539 65 65 65 119 119 119 F F F 27 540-559 85 85 85 152 152 152 F F F 28 560-579 85 85 85 152 152 152 T T T 29 580-599 84 84 84 149 149 149 F F F Max gen: 86, Max flush: 154 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 84 84 84 148 148 148 F F F 4 080-099 80 80 80 144 144 144 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 86 86 86 154 154 154 F F F 7 140-159 87 87 87 155 155 155 F F F 8 160-179 87 87 87 155 155 155 F F F 9 180-199 87 87 87 155 155 155 F F F 10 200-219 85 85 85 152 152 152 F F F 11 220-239 84 84 84 150 150 150 F F F 12 240-259 87 87 87 155 155 155 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 86 86 86 154 154 154 F F F 15 300-319 80 80 80 144 144 144 F F F 16 320-339 86 86 86 153 153 153 F F F 17 340-359 87 87 87 155 155 155 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 87 87 87 155 155 155 F F F 20 400-419 85 85 85 152 152 152 F F F 21 420-439 84 84 84 151 151 151 F F F 22 440-459 87 87 87 155 155 155 F F F 23 460-479 85 85 85 152 152 152 F F F 24 480-499 87 87 87 155 155 155 F F F 25 500-519 87 87 87 155 155 155 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 85 85 85 152 152 152 F F F 28 560-579 87 87 87 155 155 155 F F F 29 580-599 84 84 84 149 149 149 F F F Max gen: 87, Max flush: 155 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.429814364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.472897959Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.473532667Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.473582492Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.473600841Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.473618709Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539} {"msg":"Crucible ea457a3a-2b39-43e6-9d23-2843b38f77e6 has session id: 5c38cda8-af51-4549-b52f-2177fdcd18b9","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.474208901Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Upstairs opts: Upstairs UUID: ea457a3a-2b39-43e6-9d23-2843b38f77e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.474345608Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Crucible stats registered with UUID: ea457a3a-2b39-43e6-9d23-2843b38f77e6","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.474383544Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:42.474406812Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.47484012Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.474870877Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539} {"msg":"ea457a3a-2b39-43e6-9d23-2843b38f77e6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.474929441Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.474948349Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"0","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.474969707Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"0","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.474983316Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"1","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.474997205Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"1","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.475010073Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"2","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.475023702Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"2","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.475036401Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.475049549Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.475062868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:47544)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.475309434Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:47699)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.47534704Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:44779)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.475368128Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.47586866Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"2","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.475906966Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"0","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.475936913Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"1","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.476521336Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"0","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.476550683Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"2","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.47669026Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"1","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477082481Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.4772006Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"[0]R flush_numbers[0..12]: [141, 144, 155, 148, 144, 138, 154, 155, 155, 155, 152, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477298041Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"[0]R generation[0..12]: [77, 80, 87, 84, 80, 76, 86, 87, 87, 87, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477319658Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477334567Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"[1]R flush_numbers[0..12]: [141, 144, 155, 148, 144, 138, 154, 155, 155, 155, 152, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477348436Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"[1]R generation[0..12]: [77, 80, 87, 84, 80, 76, 86, 87, 87, 87, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477361664Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477379642Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"[2]R flush_numbers[0..12]: [141, 144, 155, 148, 144, 138, 154, 155, 155, 155, 152, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477393321Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"[2]R generation[0..12]: [77, 80, 87, 84, 80, 76, 86, 87, 87, 87, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.47740648Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477420049Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477432727Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477445446Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477457885Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:12:42.477470424Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"":"downstairs","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477496321Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.47751015Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"0","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477523268Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"1","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477535957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"client":"2","":"downstairs","session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477548826Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477560725Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"ea457a3a-2b39-43e6-9d23-2843b38f77e6 is now active with session: 5c38cda8-af51-4549-b52f-2177fdcd18b9","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477573094Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477587712Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539,"session_id":"5c38cda8-af51-4549-b52f-2177fdcd18b9"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:42.477599891Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 206 len 16384 data: 8 7 7 7 02/30 Flush 03/30 Flush 04/30 Write block 352 len 24576 data: 6 7 7 6 5 6 05/30 Read block 523 len 12288 06/30 Write block 316 len 40960 data: 11 9 7 3 5 7 10 9 11 11 07/30 Read block 105 len 40960 08/30 Write block 300 len 16384 data: 8 8 6 7 09/30 Write block 78 len 16384 data: 6 7 7 8 10/30 Read block 62 len 36864 11/30 Write block 346 len 28672 data: 10 11 11 8 8 8 7 12/30 Flush 13/30 Write block 285 len 4096 data: 5 14/30 Read block 325 len 24576 15/30 Read block 503 len 24576 16/30 Read block 69 len 4096 17/30 Read block 48 len 16384 18/30 Write block 547 len 28672 data: 4 6 9 10 10 11 12 19/30 Read block 85 len 20480 20/30 Read block 383 len 8192 21/30 Write block 444 len 28672 data: 10 12 11 12 10 9 10 22/30 Read block 479 len 20480 23/30 Write block 467 len 16384 data: 7 5 6 6 24/30 Read block 402 len 28672 25/30 Flush 26/30 Flush 27/30 Write block 236 len 4096 data: 7 28/30 Read block 188 len 36864 29/30 Write block 72 len 16384 data: 10 8 8 7 30/30 Write block 82 len 28672 data: 8 9 8 7 5 6 6 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:42.497279385Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1539} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 88 88 88 158 158 158 T T T 4 080-099 88 88 88 158 158 158 T T T 5 100-119 76 76 76 138 138 138 F F F 6 120-139 86 86 86 154 154 154 F F F 7 140-159 87 87 87 155 155 155 F F F 8 160-179 87 87 87 155 155 155 F F F 9 180-199 87 87 87 155 155 155 F F F 10 200-219 88 88 88 156 156 156 F F F 11 220-239 84 84 84 150 150 150 T T T 12 240-259 87 87 87 155 155 155 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 88 88 88 159 159 159 F F F 15 300-319 88 88 88 158 158 158 F F F 16 320-339 88 88 88 158 158 158 F F F 17 340-359 88 88 88 158 158 158 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 87 87 87 155 155 155 F F F 20 400-419 85 85 85 152 152 152 F F F 21 420-439 84 84 84 151 151 151 F F F 22 440-459 88 88 88 159 159 159 F F F 23 460-479 88 88 88 159 159 159 F F F 24 480-499 87 87 87 155 155 155 F F F 25 500-519 87 87 87 155 155 155 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 88 88 88 159 159 159 F F F 28 560-579 87 87 87 155 155 155 F F F 29 580-599 84 84 84 149 149 149 F F F Max gen: 88, Max flush: 159 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 89 89 89 160 160 160 F F F 4 080-099 89 89 89 160 160 160 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 86 86 86 154 154 154 F F F 7 140-159 87 87 87 155 155 155 F F F 8 160-179 87 87 87 155 155 155 F F F 9 180-199 87 87 87 155 155 155 F F F 10 200-219 88 88 88 156 156 156 F F F 11 220-239 89 89 89 160 160 160 F F F 12 240-259 87 87 87 155 155 155 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 88 88 88 159 159 159 F F F 15 300-319 88 88 88 158 158 158 F F F 16 320-339 88 88 88 158 158 158 F F F 17 340-359 88 88 88 158 158 158 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 87 87 87 155 155 155 F F F 20 400-419 85 85 85 152 152 152 F F F 21 420-439 84 84 84 151 151 151 F F F 22 440-459 88 88 88 159 159 159 F F F 23 460-479 88 88 88 159 159 159 F F F 24 480-499 87 87 87 155 155 155 F F F 25 500-519 87 87 87 155 155 155 F F F 26 520-539 65 65 65 119 119 119 F F F 27 540-559 88 88 88 159 159 159 F F F 28 560-579 87 87 87 155 155 155 F F F 29 580-599 84 84 84 149 149 149 F F F Max gen: 89, Max flush: 160 Begin loop 45 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.719419964Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.762367222Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.762964744Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.76300606Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.763023078Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.763040747Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547} {"msg":"Crucible c9768e18-ec3a-4425-bf43-9d29eee531c1 has session id: 954f55ac-1553-48c0-8b07-0b3f65474db4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.763496782Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Upstairs opts: Upstairs UUID: c9768e18-ec3a-4425-bf43-9d29eee531c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.763542918Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Crucible stats registered with UUID: c9768e18-ec3a-4425-bf43-9d29eee531c1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.763568465Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:46.763589243Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764082465Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764112432Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547} {"msg":"c9768e18-ec3a-4425-bf43-9d29eee531c1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764126061Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.76413923Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"0","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764171926Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"0","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764199074Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"1","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764223092Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"1","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764245859Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"2","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764272907Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"2","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764298564Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764328771Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764357248Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"ds_connection connected from Ok(127.0.0.1:57089)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764384286Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"ds_connection connected from Ok(127.0.0.1:45784)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764408374Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"ds_connection connected from Ok(127.0.0.1:52267)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.764433171Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.765209525Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"1","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.765236713Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"2","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.765252011Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"0","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.765894469Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"1","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.765933025Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"2","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.765960202Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"0","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.7664958Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766529897Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"[0]R flush_numbers[0..12]: [141, 144, 155, 160, 160, 138, 154, 155, 155, 155, 156, 160]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766547785Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"[0]R generation[0..12]: [77, 80, 87, 89, 89, 76, 86, 87, 87, 87, 88, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766566623Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766581302Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"[1]R flush_numbers[0..12]: [141, 144, 155, 160, 160, 138, 154, 155, 155, 155, 156, 160]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766595021Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"[1]R generation[0..12]: [77, 80, 87, 89, 89, 76, 86, 87, 87, 87, 88, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766608239Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766621838Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"[2]R flush_numbers[0..12]: [141, 144, 155, 160, 160, 138, 154, 155, 155, 155, 156, 160]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766635397Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"[2]R generation[0..12]: [77, 80, 87, 89, 89, 76, 86, 87, 87, 87, 88, 89]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766649265Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766670733Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:12:46.766695031Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766719089Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766735107Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766748606Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"":"downstairs","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766761384Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766773143Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"0","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766788762Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"1","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.76680185Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"client":"2","":"downstairs","session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766815099Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766827198Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"c9768e18-ec3a-4425-bf43-9d29eee531c1 is now active with session: 954f55ac-1553-48c0-8b07-0b3f65474db4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766839787Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766851266Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547,"session_id":"954f55ac-1553-48c0-8b07-0b3f65474db4"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:46.766863095Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 345 len 20480 data: 9 11 12 12 9 02/30 Read block 153 len 36864 03/30 Write block 486 len 32768 data: 7 8 7 7 7 7 8 10 04/30 Write block 214 len 16384 data: 8 7 7 6 05/30 Flush 06/30 Write block 234 len 36864 data: 6 7 8 9 11 11 12 12 12 07/30 Read block 497 len 32768 08/30 Read block 41 len 40960 09/30 Write block 450 len 8192 data: 11 8 10/30 Read block 342 len 4096 11/30 Write block 333 len 8192 data: 9 9 12/30 Read block 111 len 8192 13/30 Flush 14/30 Write block 588 len 40960 data: 6 9 7 6 7 8 8 6 6 6 15/30 Read block 302 len 16384 16/30 Write block 158 len 36864 data: 9 8 8 9 10 11 14 11 9 17/30 Write block 509 len 24576 data: 9 10 9 11 11 9 18/30 Write block 511 len 28672 data: 10 12 12 10 8 9 8 19/30 Read block 56 len 12288 20/30 Flush 21/30 Write block 386 len 36864 data: 8 8 9 8 7 8 9 8 5 22/30 Flush 23/30 Write block 471 len 16384 data: 6 5 5 2 24/30 Read block 533 len 8192 25/30 Write block 100 len 32768 data: 3 4 5 4 6 5 5 8 26/30 Write block 538 len 36864 data: 7 6 6 5 4 4 4 7 6 27/30 Read block 518 len 28672 28/30 Read block 13 len 4096 29/30 Read block 553 len 16384 30/30 Write block 84 len 40960 data: 9 8 6 7 7 8 9 8 7 7 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:46.787056608Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1547} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 89 89 89 160 160 160 F F F 4 080-099 89 89 89 160 160 160 T T T 5 100-119 76 76 76 138 138 138 T T T 6 120-139 86 86 86 154 154 154 F F F 7 140-159 90 90 90 163 163 163 F F F 8 160-179 90 90 90 163 163 163 F F F 9 180-199 87 87 87 155 155 155 F F F 10 200-219 90 90 90 161 161 161 F F F 11 220-239 90 90 90 162 162 162 F F F 12 240-259 90 90 90 162 162 162 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 88 88 88 159 159 159 F F F 15 300-319 88 88 88 158 158 158 F F F 16 320-339 90 90 90 162 162 162 F F F 17 340-359 90 90 90 161 161 161 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 90 90 90 164 164 164 F F F 20 400-419 85 85 85 152 152 152 F F F 21 420-439 84 84 84 151 151 151 F F F 22 440-459 90 90 90 162 162 162 F F F 23 460-479 88 88 88 159 159 159 T T T 24 480-499 90 90 90 161 161 161 F F F 25 500-519 90 90 90 163 163 163 F F F 26 520-539 65 65 65 119 119 119 T T T 27 540-559 88 88 88 159 159 159 T T T 28 560-579 87 87 87 155 155 155 F F F 29 580-599 90 90 90 163 163 163 F F F Max gen: 90, Max flush: 164 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 89 89 89 160 160 160 F F F 4 080-099 91 91 91 165 165 165 F F F 5 100-119 91 91 91 165 165 165 F F F 6 120-139 86 86 86 154 154 154 F F F 7 140-159 90 90 90 163 163 163 F F F 8 160-179 90 90 90 163 163 163 F F F 9 180-199 87 87 87 155 155 155 F F F 10 200-219 90 90 90 161 161 161 F F F 11 220-239 90 90 90 162 162 162 F F F 12 240-259 90 90 90 162 162 162 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 88 88 88 159 159 159 F F F 15 300-319 88 88 88 158 158 158 F F F 16 320-339 90 90 90 162 162 162 F F F 17 340-359 90 90 90 161 161 161 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 90 90 90 164 164 164 F F F 20 400-419 85 85 85 152 152 152 F F F 21 420-439 84 84 84 151 151 151 F F F 22 440-459 90 90 90 162 162 162 F F F 23 460-479 91 91 91 165 165 165 F F F 24 480-499 90 90 90 161 161 161 F F F 25 500-519 90 90 90 163 163 163 F F F 26 520-539 91 91 91 165 165 165 F F F 27 540-559 91 91 91 165 165 165 F F F 28 560-579 87 87 87 155 155 155 F F F 29 580-599 90 90 90 163 163 163 F F F Max gen: 91, Max flush: 165 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.02343812Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:51.065217142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.065601065Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.096554141Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.097115046Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.097146703Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.097160212Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.097174361Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555} {"msg":"Crucible 9e5b60d3-a797-491d-a3a4-8b8f8859f581 has session id: 9078466d-6f4f-4336-8b19-bb429fd8eb24","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.097759473Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Upstairs opts: Upstairs UUID: 9e5b60d3-a797-491d-a3a4-8b8f8859f581, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.097816308Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Crucible stats registered with UUID: 9e5b60d3-a797-491d-a3a4-8b8f8859f581","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.097845855Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:51.097865083Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098261105Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098289162Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555} {"msg":"9e5b60d3-a797-491d-a3a4-8b8f8859f581 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098349236Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098366465Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"0","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098388232Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"0","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098402291Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"1","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098425499Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"1","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098452516Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"2","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098478903Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"2","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098505101Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098529458Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098553696Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"ds_connection connected from Ok(127.0.0.1:42057)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098581544Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"ds_connection connected from Ok(127.0.0.1:43958)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098606101Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"ds_connection connected from Ok(127.0.0.1:61303)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.098731469Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.09923844Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"2","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.099283405Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"1","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.099372036Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"0","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.099829312Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"2","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.099869248Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"1","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.099897095Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"0","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.10036045Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100483698Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"[0]R flush_numbers[0..12]: [141, 144, 155, 160, 165, 165, 154, 163, 163, 155, 161, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100521964Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"[0]R generation[0..12]: [77, 80, 87, 89, 91, 91, 86, 90, 90, 87, 90, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100538353Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100553032Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"[1]R flush_numbers[0..12]: [141, 144, 155, 160, 165, 165, 154, 163, 163, 155, 161, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.10056714Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"[1]R generation[0..12]: [77, 80, 87, 89, 91, 91, 86, 90, 90, 87, 90, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100581099Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100595128Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"[2]R flush_numbers[0..12]: [141, 144, 155, 160, 165, 165, 154, 163, 163, 155, 161, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100608726Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"[2]R generation[0..12]: [77, 80, 87, 89, 91, 91, 86, 90, 90, 87, 90, 90]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100621955Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100639663Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100652742Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:12:51.100665701Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100683269Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100695918Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"":"downstairs","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100709176Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100720965Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"0","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100734214Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"1","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100747203Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"client":"2","":"downstairs","session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100760531Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.10077276Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"9e5b60d3-a797-491d-a3a4-8b8f8859f581 is now active with session: 9078466d-6f4f-4336-8b19-bb429fd8eb24","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100785409Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100797098Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555,"session_id":"9078466d-6f4f-4336-8b19-bb429fd8eb24"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:51.100809037Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 159 len 24576 data: 9 9 10 11 12 15 02/30 Read block 172 len 32768 03/30 Write block 465 len 12288 data: 10 8 8 04/30 Write block 191 len 24576 data: 8 9 8 6 6 7 05/30 Write block 121 len 8192 data: 5 4 06/30 Read block 142 len 32768 07/30 Read block 585 len 16384 08/30 Write block 249 len 16384 data: 9 10 10 11 09/30 Read block 341 len 20480 10/30 Read block 344 len 4096 11/30 Write block 587 len 20480 data: 7 7 10 8 7 12/30 Read block 218 len 40960 13/30 Write block 236 len 40960 data: 9 10 12 12 13 13 13 11 9 8 14/30 Write block 197 len 8192 data: 7 5 15/30 Write block 432 len 32768 data: 6 10 11 10 11 12 12 11 16/30 Read block 97 len 36864 17/30 Flush 18/30 Write block 176 len 24576 data: 11 10 9 11 13 10 19/30 Read block 124 len 8192 20/30 Write block 106 len 8192 data: 6 9 21/30 Read block 322 len 12288 22/30 Read block 235 len 32768 23/30 Write block 392 len 16384 data: 10 9 6 5 24/30 Write block 486 len 36864 data: 8 9 8 8 8 8 9 11 10 25/30 Read block 227 len 28672 26/30 Read block 508 len 40960 27/30 Read block 335 len 20480 28/30 Read block 517 len 28672 29/30 Write block 535 len 20480 data: 8 7 8 8 7 30/30 Write block 426 len 28672 data: 7 7 7 7 8 8 7 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:51.120707119Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1555} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 89 89 89 160 160 160 F F F 4 080-099 91 91 91 165 165 165 F F F 5 100-119 91 91 91 165 165 165 T T T 6 120-139 92 92 92 166 166 166 F F F 7 140-159 92 92 92 166 166 166 F F F 8 160-179 92 92 92 166 166 166 T T T 9 180-199 92 92 92 166 166 166 T T T 10 200-219 90 90 90 161 161 161 F F F 11 220-239 92 92 92 166 166 166 F F F 12 240-259 92 92 92 166 166 166 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 88 88 88 159 159 159 F F F 15 300-319 88 88 88 158 158 158 F F F 16 320-339 90 90 90 162 162 162 F F F 17 340-359 90 90 90 161 161 161 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 90 90 90 164 164 164 T T T 20 400-419 85 85 85 152 152 152 F F F 21 420-439 92 92 92 166 166 166 T T T 22 440-459 90 90 90 162 162 162 F F F 23 460-479 92 92 92 166 166 166 F F F 24 480-499 90 90 90 161 161 161 T T T 25 500-519 90 90 90 163 163 163 F F F 26 520-539 91 91 91 165 165 165 T T T 27 540-559 91 91 91 165 165 165 F F F 28 560-579 87 87 87 155 155 155 F F F 29 580-599 92 92 92 166 166 166 F F F Max gen: 92, Max flush: 166 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 89 89 89 160 160 160 F F F 4 080-099 91 91 91 165 165 165 F F F 5 100-119 93 93 93 167 167 167 F F F 6 120-139 92 92 92 166 166 166 F F F 7 140-159 92 92 92 166 166 166 F F F 8 160-179 93 93 93 167 167 167 F F F 9 180-199 93 93 93 167 167 167 F F F 10 200-219 90 90 90 161 161 161 F F F 11 220-239 92 92 92 166 166 166 F F F 12 240-259 92 92 92 166 166 166 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 88 88 88 159 159 159 F F F 15 300-319 88 88 88 158 158 158 F F F 16 320-339 90 90 90 162 162 162 F F F 17 340-359 90 90 90 161 161 161 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 93 93 93 167 167 167 F F F 20 400-419 85 85 85 152 152 152 F F F 21 420-439 93 93 93 167 167 167 F F F 22 440-459 90 90 90 162 162 162 F F F 23 460-479 92 92 92 166 166 166 F F F 24 480-499 93 93 93 167 167 167 F F F 25 500-519 90 90 90 163 163 163 F F F 26 520-539 93 93 93 167 167 167 F F F 27 540-559 91 91 91 165 165 165 F F F 28 560-579 87 87 87 155 155 155 F F F 29 580-599 92 92 92 166 166 166 F F F Max gen: 93, Max flush: 167 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.394127124Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.437118578Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.437730779Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.437780214Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.437798952Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.437816721Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563} {"msg":"Crucible 5d656944-5dd5-4358-af56-aa67cea2c651 has session id: e621b9e9-7262-4930-8d94-c0bb9e859b9d","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438262607Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Upstairs opts: Upstairs UUID: 5d656944-5dd5-4358-af56-aa67cea2c651, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438300273Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Crucible stats registered with UUID: 5d656944-5dd5-4358-af56-aa67cea2c651","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438316402Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:55.438329231Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438785666Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438826902Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563} {"msg":"5d656944-5dd5-4358-af56-aa67cea2c651 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438881407Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438906924Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"0","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438929652Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"0","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438943991Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"1","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438958729Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"1","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438971988Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"2","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438985597Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"2","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.438998225Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.439011594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.439038451Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44925)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.439064269Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:35856)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.439095966Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63938)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.439234192Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.43987666Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"0","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.439923245Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"1","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.440432846Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"2","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.44059438Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"0","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.440625507Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"1","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.440995851Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"2","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441166854Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441259365Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"[0]R flush_numbers[0..12]: [141, 144, 155, 160, 165, 167, 166, 166, 167, 167, 161, 166]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441449937Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"[0]R generation[0..12]: [77, 80, 87, 89, 91, 93, 92, 92, 93, 93, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441477564Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441499402Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"[1]R flush_numbers[0..12]: [141, 144, 155, 160, 165, 167, 166, 166, 167, 167, 161, 166]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.4415143Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"[1]R generation[0..12]: [77, 80, 87, 89, 91, 93, 92, 92, 93, 93, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441528449Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441542438Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"[2]R flush_numbers[0..12]: [141, 144, 155, 160, 165, 167, 166, 166, 167, 167, 161, 166]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441555936Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"[2]R generation[0..12]: [77, 80, 87, 89, 91, 93, 92, 92, 93, 93, 90, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441569135Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441587763Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441600622Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441613571Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.44162619Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441639008Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"":"downstairs","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441652167Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441663936Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"0","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441677185Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"1","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441691973Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"client":"2","":"downstairs","session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.44171889Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441738548Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"session_id":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"5d656944-5dd5-4358-af56-aa67cea2c651 is now active with session: e621b9e9-7262-4930-8d94-c0bb9e859b9d","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441755657Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441767436Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563,"session_id":"e621b9e9-7262-4930-8d94-c0bb9e859b9d"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:55.441779364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 97 len 36864 data: 7 6 4 4 5 6 5 7 6 02/30 Read block 390 len 36864 03/30 Write block 418 len 16384 data: 6 6 5 6 04/30 Read block 175 len 8192 05/30 Flush 06/30 Write block 347 len 28672 data: 13 13 10 9 9 8 8 07/30 Read block 473 len 32768 08/30 Flush 09/30 Write block 282 len 24576 data: 6 4 3 6 5 6 10/30 Read block 464 len 24576 11/30 Flush 12/30 Write block 562 len 28672 data: 7 4 2 2 2 4 7 13/30 Read block 198 len 12288 14/30 Flush 15/30 Write block 591 len 8192 data: 8 8 16/30 Write block 250 len 20480 data: 11 11 12 10 9 17/30 Write block 587 len 32768 data: 8 8 11 9 9 9 9 9 18/30 Read block 576 len 16384 19/30 Write block 227 len 12288 data: 7 8 9 20/30 Write block 538 len 16384 data: 9 8 7 6 21/30 Write block 66 len 16384 data: 10 10 10 11 22/30 Read block 388 len 28672 23/30 Write block 470 len 24576 data: 7 7 6 6 3 3 24/30 Write block 95 len 24576 data: 6 7 8 7 5 5 25/30 Read block 114 len 32768 26/30 Read block 275 len 40960 27/30 Write block 461 len 12288 data: 6 8 7 28/30 Write block 313 len 4096 data: 11 29/30 Write block 444 len 20480 data: 11 13 12 13 11 30/30 Read block 224 len 40960 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 2 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 15 16 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:55.461086724Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1563} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 89 89 89 160 160 160 T T T 4 080-099 94 94 94 168 168 168 T T T 5 100-119 94 94 94 168 168 168 T T T 6 120-139 92 92 92 166 166 166 F F F 7 140-159 92 92 92 166 166 166 F F F 8 160-179 93 93 93 167 167 167 F F F 9 180-199 93 93 93 167 167 167 F F F 10 200-219 90 90 90 161 161 161 F F F 11 220-239 92 92 92 166 166 166 T T T 12 240-259 92 92 92 166 166 166 T T T 13 260-279 87 87 87 155 155 155 F F F 14 280-299 94 94 94 170 170 170 F F F 15 300-319 88 88 88 158 158 158 T T T 16 320-339 90 90 90 162 162 162 F F F 17 340-359 94 94 94 169 169 169 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 93 93 93 167 167 167 F F F 20 400-419 94 94 94 168 168 168 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 90 90 90 162 162 162 T T T 23 460-479 92 92 92 166 166 166 T T T 24 480-499 93 93 93 167 167 167 F F F 25 500-519 90 90 90 163 163 163 F F F 26 520-539 93 93 93 167 167 167 T T T 27 540-559 91 91 91 165 165 165 T T T 28 560-579 94 94 94 171 171 171 F F F 29 580-599 92 92 92 166 166 166 T T T Max gen: 94, Max flush: 171 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 95 95 95 172 172 172 F F F 4 080-099 95 95 95 172 172 172 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 92 92 92 166 166 166 F F F 7 140-159 92 92 92 166 166 166 F F F 8 160-179 93 93 93 167 167 167 F F F 9 180-199 93 93 93 167 167 167 F F F 10 200-219 90 90 90 161 161 161 F F F 11 220-239 95 95 95 172 172 172 F F F 12 240-259 95 95 95 172 172 172 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 94 94 94 170 170 170 F F F 15 300-319 95 95 95 172 172 172 F F F 16 320-339 90 90 90 162 162 162 F F F 17 340-359 94 94 94 169 169 169 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 93 93 93 167 167 167 F F F 20 400-419 94 94 94 168 168 168 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 95 95 95 172 172 172 F F F 23 460-479 95 95 95 172 172 172 F F F 24 480-499 93 93 93 167 167 167 F F F 25 500-519 90 90 90 163 163 163 F F F 26 520-539 95 95 95 172 172 172 F F F 27 540-559 95 95 95 172 172 172 F F F 28 560-579 94 94 94 171 171 171 F F F 29 580-599 95 95 95 172 172 172 F F F Max gen: 95, Max flush: 172 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.730782322Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.774770909Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.775454702Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.775494728Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.775510177Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.775525005Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571} {"msg":"Crucible 69583649-7155-4986-b021-17ad765a5f49 has session id: d28600ed-7b19-4b7e-8f93-1953b488e918","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776113928Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"Upstairs opts: Upstairs UUID: 69583649-7155-4986-b021-17ad765a5f49, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776151864Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"Crucible stats registered with UUID: 69583649-7155-4986-b021-17ad765a5f49","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776168463Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:59.776181471Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776527108Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776551125Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571} {"msg":"69583649-7155-4986-b021-17ad765a5f49 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.77660544Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776627968Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"0","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776651665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"0","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776665754Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"1","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776688922Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"1","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776715089Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"2","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776740047Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"2","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776760775Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776780233Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776801871Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"ds_connection connected from Ok(127.0.0.1:43194)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776830668Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"ds_connection connected from Ok(127.0.0.1:43501)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776858835Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"ds_connection connected from Ok(127.0.0.1:49310)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.776887722Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.777561546Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"0","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.777599272Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"1","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.777672336Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"2","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778147339Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"0","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778228151Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"1","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778250259Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"2","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778719214Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778805125Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"[0]R flush_numbers[0..12]: [141, 144, 155, 172, 172, 172, 166, 166, 167, 167, 161, 172]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.77885911Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"[0]R generation[0..12]: [77, 80, 87, 95, 95, 95, 92, 92, 93, 93, 90, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778879158Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778894137Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"[1]R flush_numbers[0..12]: [141, 144, 155, 172, 172, 172, 166, 166, 167, 167, 161, 172]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778908145Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"[1]R generation[0..12]: [77, 80, 87, 95, 95, 95, 92, 92, 93, 93, 90, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778921534Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778935213Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"[2]R flush_numbers[0..12]: [141, 144, 155, 172, 172, 172, 166, 166, 167, 167, 161, 172]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778948631Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"[2]R generation[0..12]: [77, 80, 87, 95, 95, 95, 92, 92, 93, 93, 90, 95]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.77896182Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778975358Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {Wait for a query_work_queue command to finish before sending IO "msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.778998236Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.779012005Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.779024654Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.779037363Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"":"downstairs","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.779049991Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.77906158Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"0","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.779074449Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"1","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.779087088Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"client":"2","":"downstairs","session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.779099917Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.779111815Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"69583649-7155-4986-b021-17ad765a5f49 is now active with session: d28600ed-7b19-4b7e-8f93-1953b488e918","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.779124254Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.779135933Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571,"session_id":"d28600ed-7b19-4b7e-8f93-1953b488e918"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:12:59.779147792Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 232 len 36864 data: 3 4 7 8 10 11 13 13 14 02/30 Read block 127 len 28672 03/30 Write block 244 len 20480 data: 10 9 7 9 11 04/30 Write block 564 len 40960 data: 3 3 3 5 8 10 12 12 11 11 05/30 Read block 218 len 24576 06/30 Write block 492 len 32768 data: 10 12 11 10 9 7 8 8 07/30 Write block 443 len 8192 data: 11 12 08/30 Write block 131 len 12288 data: 6 7 7 09/30 Read block 471 len 36864 10/30 Flush 11/30 Write block 420 len 20480 data: 6 7 8 9 7 12/30 Write block 220 len 24576 data: 7 7 7 9 9 9 13/30 Write block 61 len 20480 data: 9 9 9 8 11 14/30 Read block 93 len 8192 15/30 Write block 66 len 12288 data: 11 11 11 16/30 Flush 17/30 Write block 487 len 40960 data: 10 9 9 9 9 11 13 12 11 10 18/30 Read block 460 len 12288 19/30 Read block 595 len 20480 20/30 Read block 96 len 28672 21/30 Read block 529 len 8192 22/30 Read block 334 len 4096 23/30 Read block 288 len 4096 24/30 Write block 222 len 32768 data: 8 10 10 10 10 8 9 10 25/30 Read block 295 len 36864 26/30 Read block 568 len 4096 27/30 Read block 332 len 12288 28/30 Read block 33 len 40960 29/30 Read block 577 len 28672 30/30 Read block 155 len 36864 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 10 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 14 13 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:12:59.79909276Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1571} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 96 96 96 174 174 174 F F F 4 080-099 95 95 95 172 172 172 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 96 96 96 173 173 173 F F F 7 140-159 92 92 92 166 166 166 F F F 8 160-179 93 93 93 167 167 167 F F F 9 180-199 93 93 93 167 167 167 F F F 10 200-219 90 90 90 161 161 161 F F F 11 220-239 96 96 96 174 174 174 T T T 12 240-259 96 96 96 173 173 173 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 94 94 94 170 170 170 F F F 15 300-319 95 95 95 172 172 172 F F F 16 320-339 90 90 90 162 162 162 F F F 17 340-359 94 94 94 169 169 169 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 93 93 93 167 167 167 F F F 20 400-419 94 94 94 168 168 168 F F F 21 420-439 96 96 96 174 174 174 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 172 172 172 F F F 24 480-499 96 96 96 173 173 173 T T T 25 500-519 90 90 90 163 163 163 F F F 26 520-539 95 95 95 172 172 172 F F F 27 540-559 95 95 95 172 172 172 F F F 28 560-579 96 96 96 173 173 173 F F F 29 580-599 95 95 95 172 172 172 F F F Max gen: 96, Max flush: 174 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 96 96 96 174 174 174 F F F 4 080-099 95 95 95 172 172 172 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 96 96 96 173 173 173 F F F 7 140-159 92 92 92 166 166 166 F F F 8 160-179 93 93 93 167 167 167 F F F 9 180-199 93 93 93 167 167 167 F F F 10 200-219 90 90 90 161 161 161 F F F 11 220-239 97 97 97 175 175 175 F F F 12 240-259 96 96 96 173 173 173 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 94 94 94 170 170 170 F F F 15 300-319 95 95 95 172 172 172 F F F 16 320-339 90 90 90 162 162 162 F F F 17 340-359 94 94 94 169 169 169 F F F 18 360-379 75 75 75 136 136 136 F F F 19 380-399 93 93 93 167 167 167 F F F 20 400-419 94 94 94 168 168 168 F F F 21 420-439 96 96 96 174 174 174 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 172 172 172 F F F 24 480-499 97 97 97 175 175 175 F F F 25 500-519 90 90 90 163 163 163 F F F 26 520-539 95 95 95 172 172 172 F F F 27 540-559 95 95 95 172 172 172 F F F 28 560-579 96 96 96 173 173 173 F F F 29 580-599 95 95 95 172 172 172 F F F Max gen: 97, Max flush: 175 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.016933166Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:04.058891081Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.059271704Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.090358386Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.091025701Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.091060968Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.091075617Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.091090535Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579} {"msg":"Crucible d07bb3ce-01c4-439d-affa-e7dc05c8a68e has session id: 41b20996-e939-4404-9dba-0eb1dfe7e0c0","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.091624863Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Upstairs opts: Upstairs UUID: d07bb3ce-01c4-439d-affa-e7dc05c8a68e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.09166128Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Crucible stats registered with UUID: d07bb3ce-01c4-439d-affa-e7dc05c8a68e","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.091679128Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:04.091692376Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092103627Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092129164Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579} {"msg":"d07bb3ce-01c4-439d-affa-e7dc05c8a68e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092193548Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092212086Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"0","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092233944Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"0","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092248033Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"1","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092262071Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"1","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.09227489Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"2","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092288929Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"2","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092301897Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092319215Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092341333Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58830)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092428395Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46375)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.09248172Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:41621)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.092601738Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.093131336Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"2","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.093169583Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"1","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.09330426Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"0","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.093740997Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"1","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.093769504Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"2","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.09402438Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"0","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094336769Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094446998Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"[0]R flush_numbers[0..12]: [141, 144, 155, 174, 172, 172, 173, 166, 167, 167, 161, 175]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094554808Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"[0]R generation[0..12]: [77, 80, 87, 96, 95, 95, 96, 92, 93, 93, 90, 97]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094575766Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094595394Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"[1]R flush_numbers[0..12]: [141, 144, 155, 174, 172, 172, 173, 166, 167, 167, 161, 175]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094610043Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"[1]R generation[0..12]: [77, 80, 87, 96, 95, 95, 96, 92, 93, 93, 90, 97]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094623721Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.09463787Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"[2]R flush_numbers[0..12]: [141, 144, 155, 174, 172, 172, 173, 166, 167, 167, 161, 175]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094651579Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"[2]R generation[0..12]: [77, 80, 87, 96, 95, 95, 96, 92, 93, 93, 90, 97]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094665187Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094679476Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094704543Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094729451Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094753258Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094777116Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"":"downstairs","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-01-28T17:13:04.094801764Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094826851Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"0","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094847249Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"1","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094864888Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"client":"2","":"downstairs","session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094878226Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094890265Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"d07bb3ce-01c4-439d-affa-e7dc05c8a68e is now active with session: 41b20996-e939-4404-9dba-0eb1dfe7e0c0","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094902894Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094914713Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579,"session_id":"41b20996-e939-4404-9dba-0eb1dfe7e0c0"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:04.094926812Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 70 len 40960 data: 8 10 11 9 9 8 7 5 7 8 02/30 Write block 359 len 16384 data: 7 7 6 5 03/30 Read block 563 len 12288 04/30 Write block 176 len 20480 data: 12 11 10 12 14 05/30 Read block 583 len 8192 06/30 Write block 216 len 12288 data: 8 7 6 07/30 Write block 351 len 36864 data: 10 9 9 8 7 6 7 9 8 08/30 Read block 150 len 4096 09/30 Write block 539 len 16384 data: 9 8 7 5 10/30 Read block 446 len 8192 11/30 Read block 423 len 4096 12/30 Write block 29 len 20480 data: 9 8 7 7 7 13/30 Read block 437 len 36864 14/30 Write block 77 len 16384 data: 6 8 9 8 15/30 Write block 89 len 4096 data: 9 16/30 Read block 226 len 36864 17/30 Write block 214 len 12288 data: 9 8 9 18/30 Read block 169 len 36864 19/30 Read block 251 len 4096 20/30 Read block 139 len 8192 21/30 Read block 397 len 36864 22/30 Write block 516 len 28672 data: 10 9 9 10 11 9 8 23/30 Write block 200 len 12288 data: 7 6 7 24/30 Read block 59 len 12288 25/30 Read block 83 len 12288 26/30 Write block 131 len 40960 data: 7 8 8 6 7 8 11 12 14 17 27/30 Write block 546 len 4096 data: 7 28/30 Read block 193 len 32768 29/30 Read block 458 len 4096 30/30 Write block 515 len 28672 data: 9 11 10 10 11 12 10 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:04.113504353Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1579} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 80 80 80 144 144 144 T T T 2 040-059 87 87 87 155 155 155 F F F 3 060-079 96 96 96 174 174 174 T T T 4 080-099 95 95 95 172 172 172 T T T 5 100-119 95 95 95 172 172 172 F F F 6 120-139 96 96 96 173 173 173 T T T 7 140-159 92 92 92 166 166 166 T T T 8 160-179 93 93 93 167 167 167 T T T 9 180-199 93 93 93 167 167 167 T T T 10 200-219 90 90 90 161 161 161 T T T 11 220-239 97 97 97 175 175 175 F F F 12 240-259 96 96 96 173 173 173 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 94 94 94 170 170 170 F F F 15 300-319 95 95 95 172 172 172 F F F 16 320-339 90 90 90 162 162 162 F F F 17 340-359 94 94 94 169 169 169 T T T 18 360-379 75 75 75 136 136 136 T T T 19 380-399 93 93 93 167 167 167 F F F 20 400-419 94 94 94 168 168 168 F F F 21 420-439 96 96 96 174 174 174 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 172 172 172 F F F 24 480-499 97 97 97 175 175 175 F F F 25 500-519 90 90 90 163 163 163 T T T 26 520-539 95 95 95 172 172 172 T T T 27 540-559 95 95 95 172 172 172 T T T 28 560-579 96 96 96 173 173 173 F F F 29 580-599 95 95 95 172 172 172 F F F Max gen: 97, Max flush: 175 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 98 98 98 176 176 176 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 98 98 98 176 176 176 F F F 4 080-099 98 98 98 176 176 176 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 98 98 98 176 176 176 F F F 8 160-179 98 98 98 176 176 176 F F F 9 180-199 98 98 98 176 176 176 F F F 10 200-219 98 98 98 176 176 176 F F F 11 220-239 97 97 97 175 175 175 F F F 12 240-259 96 96 96 173 173 173 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 94 94 94 170 170 170 F F F 15 300-319 95 95 95 172 172 172 F F F 16 320-339 90 90 90 162 162 162 F F F 17 340-359 98 98 98 176 176 176 F F F 18 360-379 98 98 98 176 176 176 F F F 19 380-399 93 93 93 167 167 167 F F F 20 400-419 94 94 94 168 168 168 F F F 21 420-439 96 96 96 174 174 174 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 172 172 172 F F F 24 480-499 97 97 97 175 175 175 F F F 25 500-519 98 98 98 176 176 176 F F F 26 520-539 98 98 98 176 176 176 F F F 27 540-559 98 98 98 176 176 176 F F F 28 560-579 96 96 96 173 173 173 F F F 29 580-599 95 95 95 172 172 172 F F F Max gen: 98, Max flush: 176 Begin loop 50 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.419008573Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.461943282Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.462619986Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.462673441Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.462691549Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.462709568Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587} {"msg":"Crucible 43d9b9e0-d0e8-4898-8303-e4f2d3d49d4f has session id: 150625ea-9f99-4626-ac76-d8b0be43d057","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.46329688Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Upstairs opts: Upstairs UUID: 43d9b9e0-d0e8-4898-8303-e4f2d3d49d4f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.463380632Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Crucible stats registered with UUID: 43d9b9e0-d0e8-4898-8303-e4f2d3d49d4f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.463412289Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:08.463428068Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.463855326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.463880843Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587} {"msg":"43d9b9e0-d0e8-4898-8303-e4f2d3d49d4f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.46392069Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.463936798Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"0","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.463959096Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"0","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.463972925Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"1","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.463987393Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"1","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.464024249Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"2","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.464053837Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"2","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.464079684Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.464107372Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.464131469Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"ds_connection connected from Ok(127.0.0.1:51321)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.464156347Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"ds_connection connected from Ok(127.0.0.1:50020)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.464183914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"ds_connection connected from Ok(127.0.0.1:55679)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.464312741Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.464861008Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"2","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.464905014Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"0","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.46493784Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"1","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.465435882Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"2","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.46556372Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"1","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.465592807Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"0","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.465931054Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466128815Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"[0]R flush_numbers[0..12]: [141, 176, 155, 176, 176, 172, 176, 176, 176, 176, 176, 175]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466168991Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"[0]R generation[0..12]: [77, 98, 87, 98, 98, 95, 98, 98, 98, 98, 98, 97]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466185379Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466199998Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"[1]R flush_numbers[0..12]: [141, 176, 155, 176, 176, 172, 176, 176, 176, 176, 176, 175]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466213936Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"[1]R generation[0..12]: [77, 98, 87, 98, 98, 95, 98, 98, 98, 98, 98, 97]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466227685Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466241294Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"[2]R flush_numbers[0..12]: [141, 176, 155, 176, 176, 172, 176, 176, 176, 176, 176, 175]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466259252Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"[2]R generation[0..12]: [77, 98, 87, 98, 98, 95, 98, 98, 98, 98, 98, 97]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.46627335Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466287359Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Max found gen is 99","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466299998Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Generation requested: 100 >= found:99","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO ":30,"time":"2025-01-28T17:13:08.466318736Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466332245Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466355003Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"":"downstairs","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.46638075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466405298Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"0","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466422396Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"1","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466436205Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"client":"2","":"downstairs","session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466458723Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.46648192Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"43d9b9e0-d0e8-4898-8303-e4f2d3d49d4f is now active with session: 150625ea-9f99-4626-ac76-d8b0be43d057","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466506868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466522286Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587,"session_id":"150625ea-9f99-4626-ac76-d8b0be43d057"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:08.466535495Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 545 len 16384 data: 8 8 5 7 02/30 Read block 468 len 24576 03/30 Write block 329 len 8192 data: 12 10 04/30 Read block 158 len 36864 05/30 Read block 259 len 16384 06/30 Read block 67 len 28672 07/30 Write block 554 len 32768 data: 14 13 11 10 9 8 7 6 08/30 Read block 528 len 28672 09/30 Read block 479 len 12288 10/30 Write block 509 len 12288 data: 10 11 11 11/30 Read block 107 len 16384 12/30 Read block 4 len 20480 13/30 Read block 433 len 40960 14/30 Write block 288 len 8192 data: 6 8 15/30 Read block 258 len 12288 16/30 Write block 385 len 4096 data: 7 17/30 Read block 80 len 4096 18/30 Read block 574 len 40960 19/30 Flush 20/30 Write block 213 len 28672 data: 9 10 9 10 8 7 6 21/30 Read block 7 len 20480 22/30 Flush 23/30 Write block 343 len 16384 data: 10 10 10 12 24/30 Write block 508 len 20480 data: 8 11 12 12 13 25/30 Flush 26/30 Write block 230 len 20480 data: 6 5 4 5 8 27/30 Write block 300 len 40960 data: 9 9 7 8 6 6 7 8 9 12 28/30 Write block 415 len 28672 data: 6 8 7 7 7 7 8 29/30 Write block 144 len 4096 data: 10 30/30 Write block 372 len 28672 data: 3 2 3 4 5 6 8 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:08.485602148Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1587} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 98 98 98 176 176 176 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 98 98 98 176 176 176 F F F 4 080-099 98 98 98 176 176 176 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 98 98 98 176 176 176 T T T 8 160-179 98 98 98 176 176 176 F F F 9 180-199 98 98 98 176 176 176 F F F 10 200-219 100 100 100 178 178 178 F F F 11 220-239 97 97 97 175 175 175 T T T 12 240-259 96 96 96 173 173 173 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 100 100 100 177 177 177 F F F 15 300-319 95 95 95 172 172 172 T T T 16 320-339 100 100 100 177 177 177 F F F 17 340-359 100 100 100 179 179 179 F F F 18 360-379 98 98 98 176 176 176 T T T 19 380-399 100 100 100 177 177 177 F F F 20 400-419 94 94 94 168 168 168 T T T 21 420-439 96 96 96 174 174 174 T T T 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 172 172 172 F F F 24 480-499 97 97 97 175 175 175 F F F 25 500-519 100 100 100 179 179 179 F F F 26 520-539 98 98 98 176 176 176 F F F 27 540-559 100 100 100 177 177 177 F F F 28 560-579 100 100 100 177 177 177 F F F 29 580-599 95 95 95 172 172 172 F F F Max gen: 100, Max flush: 179 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 141 141 141 F F F 1 020-039 98 98 98 176 176 176 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 98 98 98 176 176 176 F F F 4 080-099 98 98 98 176 176 176 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 101 101 101 180 180 180 F F F 8 160-179 98 98 98 176 176 176 F F F 9 180-199 98 98 98 176 176 176 F F F 10 200-219 100 100 100 178 178 178 F F F 11 220-239 101 101 101 180 180 180 F F F 12 240-259 96 96 96 173 173 173 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 100 100 100 177 177 177 F F F 15 300-319 101 101 101 180 180 180 F F F 16 320-339 100 100 100 177 177 177 F F F 17 340-359 100 100 100 179 179 179 F F F 18 360-379 101 101 101 180 180 180 F F F 19 380-399 100 100 100 177 177 177 F F F 20 400-419 101 101 101 180 180 180 F F F 21 420-439 101 101 101 180 180 180 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 172 172 172 F F F 24 480-499 97 97 97 175 175 175 F F F 25 500-519 100 100 100 179 179 179 F F F 26 520-539 98 98 98 176 176 176 F F F 27 540-559 100 100 100 177 177 177 F F F 28 560-579 100 100 100 177 177 177 F F F 29 580-599 95 95 95 172 172 172 F F F Max gen: 101, Max flush: 180 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.726317169Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:12.767736305Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.768114388Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.798591351Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.799163475Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.799198741Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.79921326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.799236478Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595} {"msg":"Crucible c1f0ff9a-3f32-416c-89f4-29895a9a5f79 has session id: 4f646ff9-160f-4681-8802-1cd14cc18b38","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.799776945Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"Upstairs opts: Upstairs UUID: c1f0ff9a-3f32-416c-89f4-29895a9a5f79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.799820821Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"Crucible stats registered with UUID: c1f0ff9a-3f32-416c-89f4-29895a9a5f79","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.799845378Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:12.799866596Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800313893Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800335541Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595} {"msg":"c1f0ff9a-3f32-416c-89f4-29895a9a5f79 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800404684Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800422942Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"0","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.80044442Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"0","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800457989Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"1","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800472028Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"1","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800484886Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"2","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800498795Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"2","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800514983Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800528862Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.80055077Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"ds_connection connected from Ok(127.0.0.1:64929)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800591306Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"ds_connection connected from Ok(127.0.0.1:48288)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800673628Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"ds_connection connected from Ok(127.0.0.1:37987)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.800888547Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.801310246Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"1","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.801460571Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"0","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.801555072Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"2","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.801899809Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"1","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802269033Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"0","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802417688Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"2","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802567553Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802735597Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"[0]R flush_numbers[0..12]: [141, 176, 155, 176, 176, 172, 176, 180, 176, 176, 178, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802851596Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"[0]R generation[0..12]: [77, 98, 87, 98, 98, 95, 98, 101, 98, 98, 100, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802874634Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802889482Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"[1]R flush_numbers[0..12]: [141, 176, 155, 176, 176, 172, 176, 180, 176, 176, 178, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802903021Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"[1]R generation[0..12]: [77, 98, 87, 98, 98, 95, 98, 101, 98, 98, 100, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802916339Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802930098Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"[2]R flush_numbers[0..12]: [141, 176, 155, 176, 176, 172, 176, 180, 176, 176, 178, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802943537Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"[2]R generation[0..12]: [77, 98, 87, 98, 98, 95, 98, 101, 98, 98, 100, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802956675Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802970074Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.802982683Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38Wait for a query_work_queue command to finish before sending IO "} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.803028948Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.803043367Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.803056056Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"":"downstairs","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.803069075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.803080923Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"0","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.803097082Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"1","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.803110271Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"client":"2","":"downstairs","session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.803123399Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.803135368Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"c1f0ff9a-3f32-416c-89f4-29895a9a5f79 is now active with session: 4f646ff9-160f-4681-8802-1cd14cc18b38","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.803147887Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.803159556Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595,"session_id":"4f646ff9-160f-4681-8802-1cd14cc18b38"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:12.803171505Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 224 len 28672 data: 11 11 11 9 10 11 7 02/30 Read block 77 len 24576 03/30 Read block 221 len 12288 04/30 Read block 217 len 24576 05/30 Write block 309 len 4096 data: 13 06/30 Read block 574 len 12288 07/30 Read block 542 len 28672 08/30 Write block 404 len 12288 data: 9 9 9 09/30 Read block 293 len 24576 10/30 Read block 184 len 4096 11/30 Read block 189 len 32768 12/30 Read block 488 len 16384 13/30 Flush 14/30 Write block 207 len 40960 data: 8 8 8 9 7 8 10 11 10 11 15/30 Read block 120 len 28672 16/30 Read block 570 len 24576 17/30 Read block 325 len 28672 18/30 Write block 392 len 40960 data: 11 10 7 6 5 4 5 6 7 7 19/30 Flush 20/30 Write block 12 len 16384 data: 10 10 9 10 21/30 Write block 293 len 24576 data: 8 7 5 7 7 6 22/30 Read block 551 len 28672 23/30 Read block 497 len 24576 24/30 Write block 427 len 36864 data: 8 8 8 9 9 8 11 12 11 25/30 Flush 26/30 Write block 73 len 32768 data: 10 10 9 8 7 9 10 9 27/30 Write block 197 len 36864 data: 8 6 6 8 7 8 9 10 9 28/30 Read block 9 len 12288 29/30 Read block 35 len 36864 30/30 Write block 469 len 28672 data: 7 8 8 7 7 4 4 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:12.824538434Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1595} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 183 183 183 F F F 1 020-039 98 98 98 176 176 176 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 98 98 98 176 176 176 T T T 4 080-099 98 98 98 176 176 176 T T T 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 101 101 101 180 180 180 F F F 8 160-179 98 98 98 176 176 176 F F F 9 180-199 98 98 98 176 176 176 T T T 10 200-219 102 102 102 182 182 182 T T T 11 220-239 102 102 102 181 181 181 F F F 12 240-259 96 96 96 173 173 173 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 102 102 102 183 183 183 F F F 15 300-319 102 102 102 181 181 181 F F F 16 320-339 100 100 100 177 177 177 F F F 17 340-359 100 100 100 179 179 179 F F F 18 360-379 101 101 101 180 180 180 F F F 19 380-399 102 102 102 182 182 182 F F F 20 400-419 102 102 102 182 182 182 F F F 21 420-439 102 102 102 183 183 183 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 172 172 172 T T T 24 480-499 97 97 97 175 175 175 F F F 25 500-519 100 100 100 179 179 179 F F F 26 520-539 98 98 98 176 176 176 F F F 27 540-559 100 100 100 177 177 177 F F F 28 560-579 100 100 100 177 177 177 F F F 29 580-599 95 95 95 172 172 172 F F F Max gen: 102, Max flush: 183 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 183 183 183 F F F 1 020-039 98 98 98 176 176 176 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 103 103 103 184 184 184 F F F 4 080-099 103 103 103 184 184 184 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 101 101 101 180 180 180 F F F 8 160-179 98 98 98 176 176 176 F F F 9 180-199 103 103 103 184 184 184 F F F 10 200-219 103 103 103 184 184 184 F F F 11 220-239 102 102 102 181 181 181 F F F 12 240-259 96 96 96 173 173 173 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 102 102 102 183 183 183 F F F 15 300-319 102 102 102 181 181 181 F F F 16 320-339 100 100 100 177 177 177 F F F 17 340-359 100 100 100 179 179 179 F F F 18 360-379 101 101 101 180 180 180 F F F 19 380-399 102 102 102 182 182 182 F F F 20 400-419 102 102 102 182 182 182 F F F 21 420-439 102 102 102 183 183 183 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 103 103 103 184 184 184 F F F 24 480-499 97 97 97 175 175 175 F F F 25 500-519 100 100 100 179 179 179 F F F 26 520-539 98 98 98 176 176 176 F F F 27 540-559 100 100 100 177 177 177 F F F 28 560-579 100 100 100 177 177 177 F F F 29 580-599 95 95 95 172 172 172 F F F Max gen: 103, Max flush: 184 Begin loop 52 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.088667214Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.131788865Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.132403055Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.132443911Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.13245863Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.132475018Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603} {"msg":"Crucible 24c05657-9bef-4faa-98a8-63cb03e34c44 has session id: ed18275d-d703-47ae-b591-159e0375a688","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133054452Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Upstairs opts: Upstairs UUID: 24c05657-9bef-4faa-98a8-63cb03e34c44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133095328Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Crucible stats registered with UUID: 24c05657-9bef-4faa-98a8-63cb03e34c44","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133110946Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:17.133123825Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133608458Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133641245Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603} {"msg":"24c05657-9bef-4faa-98a8-63cb03e34c44 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133694599Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133725736Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"0","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133761543Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"0","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.13379334Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"1","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133820777Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"1","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133848904Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"2","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133876832Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"2","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133905289Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133927077Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133948905Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"ds_connection connected from Ok(127.0.0.1:47402)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.133974912Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"ds_connection connected from Ok(127.0.0.1:59876)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.134002609Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"ds_connection connected from Ok(127.0.0.1:32872)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.134031477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.134544577Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"1","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.134726789Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"0","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.135153688Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"1","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.135297043Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"0","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.1354356Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"2","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.135817323Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.135860129Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136038841Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"2","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"[0]R flush_numbers[0..12]: [183, 176, 155, 184, 184, 172, 176, 180, 176, 184, 184, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136508885Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"[0]R generation[0..12]: [102, 98, 87, 103, 103, 95, 98, 101, 98, 103, 103, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136532843Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136547922Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"[1]R flush_numbers[0..12]: [183, 176, 155, 184, 184, 172, 176, 180, 176, 184, 184, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.13656186Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"[1]R generation[0..12]: [102, 98, 87, 103, 103, 95, 98, 101, 98, 103, 103, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136575469Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136589128Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"[2]R flush_numbers[0..12]: [183, 176, 155, 184, 184, 172, 176, 180, 176, 184, 184, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136602476Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"[2]R generation[0..12]: [102, 98, 87, 103, 103, 95, 98, 101, 98, 103, 103, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136615615Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136629214Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:13:17.136641963Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.13666405Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136676959Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136689438Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"":"downstairs","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136702307Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136713876Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"0","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136726744Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"1","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136739313Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"client":"2","":"downstairs","session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136752242Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136764141Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"24c05657-9bef-4faa-98a8-63cb03e34c44 is now active with session: ed18275d-d703-47ae-b591-159e0375a688","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.13677659Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136788318Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603,"session_id":"ed18275d-d703-47ae-b591-159e0375a688"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:17.136800247Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 425 len 8192 data: 6 8 03/30 Flush 04/30 Write block 357 len 20480 data: 8 10 9 8 7 05/30 Write block 388 len 8192 data: 10 9 06/30 Read block 503 len 36864 07/30 Read block 585 len 20480 08/30 Write block 322 len 8192 data: 11 10 09/30 Read block 370 len 20480 10/30 Flush 11/30 Write block 581 len 20480 data: 11 14 13 14 13 12/30 Read block 4 len 32768 13/30 Read block 435 len 28672 14/30 Flush 15/30 Write block 560 len 36864 data: 8 7 8 5 4 4 4 6 9 16/30 Write block 530 len 12288 data: 10 11 10 17/30 Write block 279 len 4096 data: 9 18/30 Flush 19/30 Write block 266 len 36864 data: 13 12 10 8 8 9 7 7 9 20/30 Write block 395 len 36864 data: 7 6 5 6 7 8 8 7 11 21/30 Write block 456 len 40960 data: 7 6 5 5 7 7 9 8 9 11 22/30 Read block 514 len 40960 23/30 Read block 79 len 12288 24/30 Flush 25/30 Write block 516 len 20480 data: 12 11 11 12 13 26/30 Write block 531 len 8192 data: 12 11 27/30 Write block 434 len 36864 data: 13 12 12 13 13 12 11 9 10 28/30 Write block 578 len 32768 data: 13 10 11 12 15 14 15 14 29/30 Write block 443 len 24576 data: 12 13 14 13 14 12 30/30 Write block 19 len 32768 data: 6 5 5 8 8 7 8 9 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:17.155112754Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1603} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 183 183 183 T T T 1 020-039 98 98 98 176 176 176 T T T 2 040-059 87 87 87 155 155 155 F F F 3 060-079 103 103 103 184 184 184 F F F 4 080-099 103 103 103 184 184 184 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 101 101 101 180 180 180 F F F 8 160-179 98 98 98 176 176 176 F F F 9 180-199 103 103 103 184 184 184 F F F 10 200-219 103 103 103 184 184 184 F F F 11 220-239 102 102 102 181 181 181 F F F 12 240-259 96 96 96 173 173 173 F F F 13 260-279 104 104 104 190 190 190 F F F 14 280-299 102 102 102 183 183 183 F F F 15 300-319 102 102 102 181 181 181 F F F 16 320-339 104 104 104 187 187 187 F F F 17 340-359 104 104 104 187 187 187 F F F 18 360-379 104 104 104 187 187 187 F F F 19 380-399 104 104 104 190 190 190 F F F 20 400-419 104 104 104 190 190 190 F F F 21 420-439 104 104 104 186 186 186 T T T 22 440-459 104 104 104 190 190 190 T T T 23 460-479 104 104 104 190 190 190 F F F 24 480-499 97 97 97 175 175 175 F F F 25 500-519 100 100 100 179 179 179 T T T 26 520-539 104 104 104 189 189 189 T T T 27 540-559 100 100 100 177 177 177 F F F 28 560-579 104 104 104 189 189 189 T T T 29 580-599 104 104 104 188 188 188 T T T Max gen: 104, Max flush: 190 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 191 191 191 F F F 1 020-039 105 105 105 191 191 191 F F F 2 040-059 87 87 87 155 155 155 F F F 3 060-079 103 103 103 184 184 184 F F F 4 080-099 103 103 103 184 184 184 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 101 101 101 180 180 180 F F F 8 160-179 98 98 98 176 176 176 F F F 9 180-199 103 103 103 184 184 184 F F F 10 200-219 103 103 103 184 184 184 F F F 11 220-239 102 102 102 181 181 181 F F F 12 240-259 96 96 96 173 173 173 F F F 13 260-279 104 104 104 190 190 190 F F F 14 280-299 102 102 102 183 183 183 F F F 15 300-319 102 102 102 181 181 181 F F F 16 320-339 104 104 104 187 187 187 F F F 17 340-359 104 104 104 187 187 187 F F F 18 360-379 104 104 104 187 187 187 F F F 19 380-399 104 104 104 190 190 190 F F F 20 400-419 104 104 104 190 190 190 F F F 21 420-439 105 105 105 191 191 191 F F F 22 440-459 105 105 105 191 191 191 F F F 23 460-479 104 104 104 190 190 190 F F F 24 480-499 97 97 97 175 175 175 F F F 25 500-519 105 105 105 191 191 191 F F F 26 520-539 105 105 105 191 191 191 F F F 27 540-559 100 100 100 177 177 177 F F F 28 560-579 105 105 105 191 191 191 F F F 29 580-599 105 105 105 191 191 191 F F F Max gen: 105, Max flush: 191 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.401981154Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.444902685Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.445481208Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.445520564Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.445535183Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.445549632Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611} {"msg":"Crucible 026e5865-f9c8-40a7-9f32-4bc7c470f060 has session id: abe00323-3403-4804-b3a2-08b4d64ab609","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446096788Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Upstairs opts: Upstairs UUID: 026e5865-f9c8-40a7-9f32-4bc7c470f060, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446156582Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Crucible stats registered with UUID: 026e5865-f9c8-40a7-9f32-4bc7c470f060","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.44618158Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:21.446209337Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446677032Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446747105Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611} {"msg":"026e5865-f9c8-40a7-9f32-4bc7c470f060 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446773293Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446786581Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"0","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446807799Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"0","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446821068Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"1","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446844846Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"1","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446869593Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"2","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446895481Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"2","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446916089Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446941186Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446967384Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"ds_connection connected from Ok(127.0.0.1:42084)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.446995331Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"ds_connection connected from Ok(127.0.0.1:52291)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.447119589Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"ds_connection connected from Ok(127.0.0.1:64146)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.447157675Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.447698152Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"0","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.44772623Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"1","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.448263987Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"0","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.448365827Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"1","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.448841381Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.448880267Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.449089787Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"2","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.449555771Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"2","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"[0]R flush_numbers[0..12]: [191, 191, 155, 184, 184, 172, 176, 180, 176, 184, 184, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450014837Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"[0]R generation[0..12]: [105, 105, 87, 103, 103, 95, 98, 101, 98, 103, 103, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450037535Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450052203Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"[1]R flush_numbers[0..12]: [191, 191, 155, 184, 184, 172, 176, 180, 176, 184, 184, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450066152Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"[1]R generation[0..12]: [105, 105, 87, 103, 103, 95, 98, 101, 98, 103, 103, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450079511Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450093119Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"[2]R flush_numbers[0..12]: [191, 191, 155, 184, 184, 172, 176, 180, 176, 184, 184, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450110008Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"[2]R generation[0..12]: [105, 105, 87, 103, 103, 95, 98, 101, 98, 103, 103, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450123766Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450137165Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450149644Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:13:21.450162672Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450181601Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450194179Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"":"downstairs","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450207078Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450218627Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"0","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450231386Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"1","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450243995Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"client":"2","":"downstairs","session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450256773Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450268642Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"026e5865-f9c8-40a7-9f32-4bc7c470f060 is now active with session: abe00323-3403-4804-b3a2-08b4d64ab609","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450280971Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.45029269Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611,"session_id":"abe00323-3403-4804-b3a2-08b4d64ab609"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:21.450306638Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 219 len 4096 data: 7 02/30 Write block 83 len 12288 data: 10 10 9 03/30 Write block 390 len 40960 data: 8 9 12 11 8 8 7 6 7 8 04/30 Write block 462 len 20480 data: 10 9 10 12 9 05/30 Write block 519 len 32768 data: 13 14 11 9 8 8 6 4 06/30 Write block 5 len 16384 data: 4 4 7 8 07/30 Write block 353 len 28672 data: 10 9 8 7 9 11 10 08/30 Write block 540 len 4096 data: 9 09/30 Write block 496 len 8192 data: 11 8 10/30 Write block 50 len 4096 data: 5 11/30 Write block 407 len 20480 data: 8 10 8 8 6 12/30 Read block 494 len 24576 13/30 Flush 14/30 Write block 172 len 40960 data: 13 13 13 14 13 12 11 13 15 11 15/30 Read block 239 len 20480 16/30 Read block 525 len 20480 17/30 Write block 498 len 36864 data: 9 9 7 6 5 7 9 11 10 18/30 Read block 221 len 24576 19/30 Write block 512 len 12288 data: 14 13 11 20/30 Flush 21/30 Write block 229 len 40960 data: 12 8 6 5 6 9 9 11 12 14 22/30 Read block 436 len 32768 23/30 Flush 24/30 Write block 301 len 32768 data: 10 8 9 7 7 8 9 10 25/30 Write block 339 len 8192 data: 8 7 26/30 Write block 585 len 8192 data: 15 9 27/30 Read block 308 len 16384 28/30 Read block 265 len 28672 29/30 Read block 80 len 20480 30/30 Read block 131 len 32768 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:21.4679549Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1611} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 192 192 192 F F F 1 020-039 105 105 105 191 191 191 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 103 103 103 184 184 184 F F F 4 080-099 106 106 106 192 192 192 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 101 101 101 180 180 180 F F F 8 160-179 106 106 106 193 193 193 F F F 9 180-199 106 106 106 193 193 193 F F F 10 200-219 106 106 106 192 192 192 F F F 11 220-239 106 106 106 194 194 194 F F F 12 240-259 96 96 96 173 173 173 F F F 13 260-279 104 104 104 190 190 190 F F F 14 280-299 102 102 102 183 183 183 F F F 15 300-319 102 102 102 181 181 181 T T T 16 320-339 104 104 104 187 187 187 T T T 17 340-359 106 106 106 192 192 192 T T T 18 360-379 104 104 104 187 187 187 F F F 19 380-399 106 106 106 192 192 192 F F F 20 400-419 106 106 106 192 192 192 F F F 21 420-439 105 105 105 191 191 191 F F F 22 440-459 105 105 105 191 191 191 F F F 23 460-479 106 106 106 192 192 192 F F F 24 480-499 106 106 106 193 193 193 F F F 25 500-519 106 106 106 193 193 193 F F F 26 520-539 106 106 106 192 192 192 F F F 27 540-559 106 106 106 192 192 192 F F F 28 560-579 105 105 105 191 191 191 F F F 29 580-599 105 105 105 191 191 191 T T T Max gen: 106, Max flush: 194 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 192 192 192 F F F 1 020-039 105 105 105 191 191 191 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 103 103 103 184 184 184 F F F 4 080-099 106 106 106 192 192 192 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 101 101 101 180 180 180 F F F 8 160-179 106 106 106 193 193 193 F F F 9 180-199 106 106 106 193 193 193 F F F 10 200-219 106 106 106 192 192 192 F F F 11 220-239 106 106 106 194 194 194 F F F 12 240-259 96 96 96 173 173 173 F F F 13 260-279 104 104 104 190 190 190 F F F 14 280-299 102 102 102 183 183 183 F F F 15 300-319 107 107 107 195 195 195 F F F 16 320-339 107 107 107 195 195 195 F F F 17 340-359 107 107 107 195 195 195 F F F 18 360-379 104 104 104 187 187 187 F F F 19 380-399 106 106 106 192 192 192 F F F 20 400-419 106 106 106 192 192 192 F F F 21 420-439 105 105 105 191 191 191 F F F 22 440-459 105 105 105 191 191 191 F F F 23 460-479 106 106 106 192 192 192 F F F 24 480-499 106 106 106 193 193 193 F F F 25 500-519 106 106 106 193 193 193 F F F 26 520-539 106 106 106 192 192 192 F F F 27 540-559 106 106 106 192 192 192 F F F 28 560-579 105 105 105 191 191 191 F F F 29 580-599 107 107 107 195 195 195 F F F Max gen: 107, Max flush: 195 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.698579043Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.741415161Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.741991736Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.742039451Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.742060449Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.742076207Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619} {"msg":"Crucible 8cb180b0-fcea-4610-9f7c-606da9b36809 has session id: f0d95dad-0eb4-4fe4-8e5e-196638af3ed8","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.7425644Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Upstairs opts: Upstairs UUID: 8cb180b0-fcea-4610-9f7c-606da9b36809, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.742624354Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Crucible stats registered with UUID: 8cb180b0-fcea-4610-9f7c-606da9b36809","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.742647561Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:25.7426618Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743060971Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743082939Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619} {"msg":"8cb180b0-fcea-4610-9f7c-606da9b36809 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.74317361Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743196358Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"0","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743215266Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"0","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743228815Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"1","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743242544Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"1","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743256422Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"2","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743270351Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"2","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743288629Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743323036Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743350473Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"ds_connection connected from Ok(127.0.0.1:53231)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.74338758Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"ds_connection connected from Ok(127.0.0.1:60840)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743414007Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"ds_connection connected from Ok(127.0.0.1:47325)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.743518887Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.744089621Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"2","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.744118678Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"0","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.744778754Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"2","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.744806581Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"0","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.744854797Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"1","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.745369396Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.745469667Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"1","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.745489725Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"[0]R flush_numbers[0..12]: [192, 191, 192, 184, 192, 172, 176, 180, 193, 193, 192, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.745982407Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"[0]R generation[0..12]: [106, 105, 106, 103, 106, 95, 98, 101, 106, 106, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746004325Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746018973Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"[1]R flush_numbers[0..12]: [192, 191, 192, 184, 192, 172, 176, 180, 193, 193, 192, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746032962Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"[1]R generation[0..12]: [106, 105, 106, 103, 106, 95, 98, 101, 106, 106, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.74605714Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746073898Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"[2]R flush_numbers[0..12]: [192, 191, 192, 184, 192, 172, 176, 180, 193, 193, 192, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746087567Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"[2]R generation[0..12]: [106, 105, 106, 103, 106, 95, 98, 101, 106, 106, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746100885Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746114504Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746127153Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746146121Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.74615897Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746178088Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"":"downstairs","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746192706Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746204535Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"0","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746219974Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"1","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746233272Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"client":"2","":"downstairs","session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746246211Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.74625829Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"8cb180b0-fcea-4610-9f7c-606da9b36809 is now active with session: f0d95dad-0eb4-4fe4-8e5e-196638af3ed8","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746270759Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:25.746282438Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619,"session_id":"f0d95dad-0eb4-4fe4-8e5e-196638af3ed8"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-01-28T17:13:25.746294226Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 438 len 4096 data: 14 03/30 Write block 41 len 36864 data: 6 6 7 7 5 5 7 5 5 04/30 Read block 74 len 16384 05/30 Read block 555 len 40960 06/30 Flush 07/30 Write block 438 len 16384 data: 15 13 12 10 08/30 Write block 532 len 8192 data: 12 11 09/30 Read block 510 len 8192 10/30 Read block 352 len 36864 11/30 Write block 210 len 8192 data: 10 8 12/30 Read block 448 len 36864 13/30 Read block 54 len 24576 14/30 Read block 239 len 32768 15/30 Read block 585 len 8192 16/30 Write block 298 len 32768 data: 7 6 10 11 9 10 8 8 17/30 Read block 344 len 36864 18/30 Read block 108 len 8192 19/30 Read block 123 len 8192 20/30 Write block 294 len 32768 data: 8 6 8 8 8 7 11 12 21/30 Write block 453 len 4096 data: 8 22/30 Read block 485 len 24576 23/30 Read block 225 len 24576 24/30 Write block 319 len 40960 data: 4 6 8 12 11 12 12 11 12 13 25/30 Write block 502 len 12288 data: 6 8 10 26/30 Read block 325 len 8192 27/30 Read block 289 len 8192 28/30 Read block 259 len 24576 29/30 Read block 244 len 20480 30/30 Read block 486 len 36864 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 4 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:25.764859419Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1619} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 192 192 192 F F F 1 020-039 105 105 105 191 191 191 F F F 2 040-059 108 108 108 197 197 197 F F F 3 060-079 103 103 103 184 184 184 F F F 4 080-099 106 106 106 192 192 192 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 101 101 101 180 180 180 F F F 8 160-179 106 106 106 193 193 193 F F F 9 180-199 106 106 106 193 193 193 F F F 10 200-219 106 106 106 192 192 192 T T T 11 220-239 106 106 106 194 194 194 F F F 12 240-259 96 96 96 173 173 173 F F F 13 260-279 104 104 104 190 190 190 F F F 14 280-299 102 102 102 183 183 183 T T T 15 300-319 107 107 107 195 195 195 T T T 16 320-339 107 107 107 195 195 195 T T T 17 340-359 107 107 107 195 195 195 F F F 18 360-379 104 104 104 187 187 187 F F F 19 380-399 106 106 106 192 192 192 F F F 20 400-419 106 106 106 192 192 192 F F F 21 420-439 108 108 108 197 197 197 T T T 22 440-459 105 105 105 191 191 191 T T T 23 460-479 106 106 106 192 192 192 F F F 24 480-499 106 106 106 193 193 193 F F F 25 500-519 106 106 106 193 193 193 T T T 26 520-539 106 106 106 192 192 192 T T T 27 540-559 106 106 106 192 192 192 F F F 28 560-579 105 105 105 191 191 191 F F F 29 580-599 107 107 107 195 195 195 F F F Max gen: 108, Max flush: 197 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 192 192 192 F F F 1 020-039 105 105 105 191 191 191 F F F 2 040-059 108 108 108 197 197 197 F F F 3 060-079 103 103 103 184 184 184 F F F 4 080-099 106 106 106 192 192 192 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 101 101 101 180 180 180 F F F 8 160-179 106 106 106 193 193 193 F F F 9 180-199 106 106 106 193 193 193 F F F 10 200-219 109 109 109 198 198 198 F F F 11 220-239 106 106 106 194 194 194 F F F 12 240-259 96 96 96 173 173 173 F F F 13 260-279 104 104 104 190 190 190 F F F 14 280-299 109 109 109 198 198 198 F F F 15 300-319 109 109 109 198 198 198 F F F 16 320-339 109 109 109 198 198 198 F F F 17 340-359 107 107 107 195 195 195 F F F 18 360-379 104 104 104 187 187 187 F F F 19 380-399 106 106 106 192 192 192 F F F 20 400-419 106 106 106 192 192 192 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 109 109 109 198 198 198 F F F 23 460-479 106 106 106 192 192 192 F F F 24 480-499 106 106 106 193 193 193 F F F 25 500-519 109 109 109 198 198 198 F F F 26 520-539 109 109 109 198 198 198 F F F 27 540-559 106 106 106 192 192 192 F F F 28 560-579 105 105 105 191 191 191 F F F 29 580-599 107 107 107 195 195 195 F F F Max gen: 109, Max flush: 198 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.016970439Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.063656423Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.064343386Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.064384832Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.064399801Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.06441429Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627} {"msg":"Crucible cd274d1a-f1cc-44bb-8ef0-7b0a695e3092 has session id: 1e3b833c-dea7-43fe-aeff-57e241eb4c1d","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.064997963Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Upstairs opts: Upstairs UUID: cd274d1a-f1cc-44bb-8ef0-7b0a695e3092, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065032869Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Crucible stats registered with UUID: cd274d1a-f1cc-44bb-8ef0-7b0a695e3092","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065047938Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:30.065060457Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065492494Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065516932Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627} {"msg":"cd274d1a-f1cc-44bb-8ef0-7b0a695e3092 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065574837Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065601644Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"0","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065624802Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"0","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.06563935Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"1","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065660238Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"1","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065685046Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"2","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065711043Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"2","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065733391Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065754589Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065769098Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:42973)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065792325Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46000)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065818653Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:64342)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.065992506Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.066516105Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"0","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.066548052Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"2","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.067156852Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"1","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.067308988Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"0","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.067340075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"2","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.067702429Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"1","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068034097Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068130808Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"[0]R flush_numbers[0..12]: [192, 191, 197, 184, 192, 172, 176, 180, 193, 193, 198, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068243197Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"[0]R generation[0..12]: [106, 105, 108, 103, 106, 95, 98, 101, 106, 106, 109, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068264125Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068279103Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"[1]R flush_numbers[0..12]: [192, 191, 197, 184, 192, 172, 176, 180, 193, 193, 198, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068292962Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"[1]R generation[0..12]: [106, 105, 108, 103, 106, 95, 98, 101, 106, 106, 109, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.06831376Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068340387Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"[2]R flush_numbers[0..12]: [192, 191, 197, 184, 192, 172, 176, 180, 193, 193, 198, 194]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:13:30.068367774Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"[2]R generation[0..12]: [106, 105, 108, 103, 106, 95, 98, 101, 106, 106, 109, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068396962Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.06841689Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068430259Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068446557Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068469745Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068494462Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"":"downstairs","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068512891Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068528499Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"0","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068552057Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"1","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068576404Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"client":"2","":"downstairs","session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068598792Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068612031Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"cd274d1a-f1cc-44bb-8ef0-7b0a695e3092 is now active with session: 1e3b833c-dea7-43fe-aeff-57e241eb4c1d","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068628829Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068652957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627,"session_id":"1e3b833c-dea7-43fe-aeff-57e241eb4c1d"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:30.068674035Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 259 len 4096 data: 7 03/30 Write block 304 len 24576 data: 9 9 9 10 11 14 04/30 Read block 546 len 12288 05/30 Read block 137 len 16384 06/30 Write block 537 len 20480 data: 9 10 10 10 8 07/30 Read block 127 len 20480 08/30 Flush 09/30 Write block 152 len 28672 data: 4 3 5 5 5 6 10 10/30 Read block 427 len 24576 11/30 Read block 213 len 40960 12/30 Read block 410 len 24576 13/30 Write block 588 len 24576 data: 9 12 10 10 10 10 14/30 Write block 246 len 12288 data: 8 10 12 15/30 Flush 16/30 Write block 520 len 40960 data: 15 12 10 9 9 7 5 3 3 7 17/30 Write block 583 len 20480 data: 15 16 16 10 9 18/30 Read block 462 len 8192 19/30 Write block 69 len 28672 data: 12 9 11 12 11 11 10 20/30 Read block 189 len 16384 21/30 Write block 349 len 8192 data: 11 10 22/30 Write block 163 len 24576 data: 13 16 12 10 11 12 23/30 Write block 323 len 32768 data: 12 13 13 12 13 14 13 11 24/30 Flush 25/30 Write block 71 len 8192 data: 12 13 26/30 Write block 194 len 32768 data: 7 7 8 9 7 7 9 8 27/30 Write block 203 len 36864 data: 10 11 10 9 9 9 9 11 9 28/30 Read block 402 len 20480 29/30 Write block 497 len 28672 data: 9 10 10 8 7 7 9 30/30 Read block 291 len 32768 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:30.087793553Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1627} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 192 192 192 F F F 1 020-039 105 105 105 191 191 191 F F F 2 040-059 108 108 108 197 197 197 F F F 3 060-079 110 110 110 202 202 202 T T T 4 080-099 106 106 106 192 192 192 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 110 110 110 201 201 201 F F F 8 160-179 110 110 110 202 202 202 F F F 9 180-199 106 106 106 193 193 193 T T T 10 200-219 109 109 109 198 198 198 T T T 11 220-239 106 106 106 194 194 194 F F F 12 240-259 110 110 110 201 201 201 F F F 13 260-279 104 104 104 190 190 190 F F F 14 280-299 109 109 109 198 198 198 F F F 15 300-319 110 110 110 200 200 200 F F F 16 320-339 110 110 110 202 202 202 F F F 17 340-359 110 110 110 202 202 202 F F F 18 360-379 104 104 104 187 187 187 F F F 19 380-399 106 106 106 192 192 192 F F F 20 400-419 106 106 106 192 192 192 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 109 109 109 198 198 198 F F F 23 460-479 106 106 106 192 192 192 F F F 24 480-499 106 106 106 193 193 193 T T T 25 500-519 109 109 109 198 198 198 T T T 26 520-539 110 110 110 202 202 202 F F F 27 540-559 110 110 110 200 200 200 F F F 28 560-579 105 105 105 191 191 191 F F F 29 580-599 110 110 110 202 202 202 F F F Max gen: 110, Max flush: 202 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 192 192 192 F F F 1 020-039 105 105 105 191 191 191 F F F 2 040-059 108 108 108 197 197 197 F F F 3 060-079 111 111 111 203 203 203 F F F 4 080-099 106 106 106 192 192 192 F F F 5 100-119 95 95 95 172 172 172 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 110 110 110 201 201 201 F F F 8 160-179 110 110 110 202 202 202 F F F 9 180-199 111 111 111 203 203 203 F F F 10 200-219 111 111 111 203 203 203 F F F 11 220-239 106 106 106 194 194 194 F F F 12 240-259 110 110 110 201 201 201 F F F 13 260-279 104 104 104 190 190 190 F F F 14 280-299 109 109 109 198 198 198 F F F 15 300-319 110 110 110 200 200 200 F F F 16 320-339 110 110 110 202 202 202 F F F 17 340-359 110 110 110 202 202 202 F F F 18 360-379 104 104 104 187 187 187 F F F 19 380-399 106 106 106 192 192 192 F F F 20 400-419 106 106 106 192 192 192 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 109 109 109 198 198 198 F F F 23 460-479 106 106 106 192 192 192 F F F 24 480-499 111 111 111 203 203 203 F F F 25 500-519 111 111 111 203 203 203 F F F 26 520-539 110 110 110 202 202 202 F F F 27 540-559 110 110 110 200 200 200 F F F 28 560-579 105 105 105 191 191 191 F F F 29 580-599 110 110 110 202 202 202 F F F Max gen: 111, Max flush: 203 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.324341569Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.367434173Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.368098478Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.368138244Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.368153413Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.368168021Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635} {"msg":"Crucible 02f081ad-83e8-4194-9d05-8089fd7199f0 has session id: e53ed727-16b1-4333-87a1-540047d898cb","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.368765793Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"Upstairs opts: Upstairs UUID: 02f081ad-83e8-4194-9d05-8089fd7199f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.36880272Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"Crucible stats registered with UUID: 02f081ad-83e8-4194-9d05-8089fd7199f0","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.368820238Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:34.368832737Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369317799Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369349196Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635} {"msg":"02f081ad-83e8-4194-9d05-8089fd7199f0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.36941554Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369442847Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"0","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369464145Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"0","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369480643Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"1","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369549197Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"1","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369583504Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"2","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.36961349Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"2","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369642068Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369668805Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369698462Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:49856)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369724Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"io task","client":"2","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48830)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369751027Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"io task","client":"0","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:53926)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.369904392Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"io task","client":"1","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.370314172Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"0","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.370432791Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"1","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.370813014Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"0","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.370911324Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"1","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.370968679Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"2","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.371389288Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.371414065Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.371622515Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"2","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"[0]R flush_numbers[0..12]: [192, 191, 197, 203, 192, 172, 176, 201, 202, 203, 203, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.37208529Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"[0]R generation[0..12]: [106, 105, 108, 111, 106, 95, 98, 110, 110, 111, 111, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372109538Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372127656Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"[1]R flush_numbers[0..12]: [192, 191, 197, 203, 192, 172, 176, 201, 202, 203, 203, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372142094Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"[1]R generation[0..12]: [106, 105, 108, 111, 106, 95, 98, 110, 110, 111, 111, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372155473Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372169082Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"[2]R flush_numbers[0..12]: [192, 191, 197, 203, 192, 172, 176, 201, 202, 203, 203, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372182591Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"[2]R generation[0..12]: [106, 105, 108, 111, 106, 95, 98, 110, 110, 111, 111, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372195769Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372209298Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msgWait for a query_work_queue command to finish before sending IO ":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372228036Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372241075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372253713Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372266182Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"":"downstairs","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372278851Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.37229049Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"0","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372303329Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"1","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372317847Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"client":"2","":"downstairs","session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372330906Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372342785Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"02f081ad-83e8-4194-9d05-8089fd7199f0 is now active with session: e53ed727-16b1-4333-87a1-540047d898cb","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372355134Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2025-01-28T17:13:34.372370152Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635,"session_id":"e53ed727-16b1-4333-87a1-540047d898cb"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:34.372382211Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 487 len 4096 data: 11 03/30 Write block 73 len 4096 data: 12 04/30 Write block 555 len 36864 data: 14 12 11 10 9 9 8 9 6 05/30 Write block 384 len 16384 data: 9 8 9 9 06/30 Write block 104 len 12288 data: 8 7 7 07/30 Read block 99 len 24576 08/30 Write block 40 len 12288 data: 8 7 7 09/30 Read block 324 len 32768 10/30 Write block 583 len 8192 data: 16 17 11/30 Read block 536 len 4096 12/30 Write block 192 len 4096 data: 10 13/30 Write block 73 len 20480 data: 13 12 11 9 8 14/30 Flush 15/30 Write block 143 len 32768 data: 12 11 9 8 7 5 6 6 16/30 Write block 487 len 16384 data: 12 10 10 10 17/30 Write block 310 len 12288 data: 15 13 11 18/30 Read block 42 len 4096 19/30 Read block 441 len 8192 20/30 Flush 21/30 Write block 557 len 40960 data: 12 11 10 10 9 10 7 5 5 5 22/30 Read block 179 len 40960 23/30 Write block 183 len 12288 data: 12 10 10 24/30 Read block 467 len 8192 25/30 Write block 515 len 20480 data: 10 13 12 12 14 26/30 Write block 190 len 40960 data: 8 9 11 9 8 8 9 10 8 8 27/30 Write block 368 len 36864 data: 6 5 5 5 4 3 4 5 6 28/30 Write block 550 len 36864 data: 11 11 12 13 15 15 13 13 12 29/30 Write block 313 len 16384 data: 12 11 13 12 30/30 Write block 322 len 4096 data: 13 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:34.388575314Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1635} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 192 192 192 F F F 1 020-039 105 105 105 191 191 191 F F F 2 040-059 112 112 112 205 205 205 F F F 3 060-079 112 112 112 205 205 205 F F F 4 080-099 106 106 106 192 192 192 F F F 5 100-119 112 112 112 205 205 205 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 112 112 112 206 206 206 F F F 8 160-179 110 110 110 202 202 202 F F F 9 180-199 112 112 112 205 205 205 T T T 10 200-219 111 111 111 203 203 203 F F F 11 220-239 106 106 106 194 194 194 F F F 12 240-259 110 110 110 201 201 201 F F F 13 260-279 104 104 104 190 190 190 F F F 14 280-299 109 109 109 198 198 198 F F F 15 300-319 112 112 112 206 206 206 T T T 16 320-339 110 110 110 202 202 202 T T T 17 340-359 110 110 110 202 202 202 F F F 18 360-379 104 104 104 187 187 187 T T T 19 380-399 112 112 112 205 205 205 F F F 20 400-419 106 106 106 192 192 192 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 109 109 109 198 198 198 F F F 23 460-479 106 106 106 192 192 192 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 111 111 111 203 203 203 T T T 26 520-539 110 110 110 202 202 202 F F F 27 540-559 112 112 112 205 205 205 T T T 28 560-579 112 112 112 205 205 205 T T T 29 580-599 112 112 112 205 205 205 F F F Max gen: 112, Max flush: 206 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 192 192 192 F F F 1 020-039 105 105 105 191 191 191 F F F 2 040-059 112 112 112 205 205 205 F F F 3 060-079 112 112 112 205 205 205 F F F 4 080-099 106 106 106 192 192 192 F F F 5 100-119 112 112 112 205 205 205 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 112 112 112 206 206 206 F F F 8 160-179 110 110 110 202 202 202 F F F 9 180-199 113 113 113 207 207 207 F F F 10 200-219 111 111 111 203 203 203 F F F 11 220-239 106 106 106 194 194 194 F F F 12 240-259 110 110 110 201 201 201 F F F 13 260-279 104 104 104 190 190 190 F F F 14 280-299 109 109 109 198 198 198 F F F 15 300-319 113 113 113 207 207 207 F F F 16 320-339 113 113 113 207 207 207 F F F 17 340-359 110 110 110 202 202 202 F F F 18 360-379 113 113 113 207 207 207 F F F 19 380-399 112 112 112 205 205 205 F F F 20 400-419 106 106 106 192 192 192 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 109 109 109 198 198 198 F F F 23 460-479 106 106 106 192 192 192 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 113 113 113 207 207 207 F F F 26 520-539 110 110 110 202 202 202 F F F 27 540-559 113 113 113 207 207 207 F F F 28 560-579 113 113 113 207 207 207 F F F 29 580-599 112 112 112 205 205 205 F F F Max gen: 113, Max flush: 207 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.630069847Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:38.672721084Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.673118386Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.704590311Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.705190073Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.705223659Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.705237178Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.705251107Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643} {"msg":"Crucible 7d3b0d4e-20bb-44fb-a730-74d0d199f936 has session id: 27f33872-93f3-42de-b800-067c2fbc4a72","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.705809082Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Upstairs opts: Upstairs UUID: 7d3b0d4e-20bb-44fb-a730-74d0d199f936, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.705855948Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Crucible stats registered with UUID: 7d3b0d4e-20bb-44fb-a730-74d0d199f936","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.705879586Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:38.705893824Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706276637Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706309404Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643} {"msg":"7d3b0d4e-20bb-44fb-a730-74d0d199f936 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706362188Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706392036Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"0","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706414074Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"0","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706428062Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"1","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.70645406Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"1","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706476557Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"2","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706490696Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"2","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706512984Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706539141Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706563739Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"ds_connection connected from Ok(127.0.0.1:52613)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706590786Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"io task","client":"0","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"ds_connection connected from Ok(127.0.0.1:55947)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706619074Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"io task","client":"2","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"ds_connection connected from Ok(127.0.0.1:38422)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.706840272Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"io task","client":"1","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.707437144Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"1","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.707466381Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"2","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.707485509Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"0","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708040665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"1","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708172682Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"0","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708201309Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"2","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708647386Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708752736Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"[0]R flush_numbers[0..12]: [192, 191, 205, 205, 192, 205, 176, 206, 202, 207, 203, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708830948Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"[0]R generation[0..12]: [106, 105, 112, 112, 106, 112, 98, 112, 110, 113, 111, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708850686Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708864915Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"[1]R flush_numbers[0..12]: [192, 191, 205, 205, 192, 205, 176, 206, 202, 207, 203, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708878574Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"[1]R generation[0..12]: [106, 105, 112, 112, 106, 112, 98, 112, 110, 113, 111, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708891822Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708905501Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"[2]R flush_numbers[0..12]: [192, 191, 205, 205, 192, 205, 176, 206, 202, 207, 203, 194]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.70891914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"[2]R generation[0..12]: [106, 105, 112, 112, 106, 112, 98, 112, 110, 113, 111, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708932538Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708950916Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708963995Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:13:38.708976844Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.708993462Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.709006001Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"":"downstairs","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.70901877Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.709030429Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"0","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.709043478Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"1","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.709056176Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"client":"2","":"downstairs","session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.709069035Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.709080914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"7d3b0d4e-20bb-44fb-a730-74d0d199f936 is now active with session: 27f33872-93f3-42de-b800-067c2fbc4a72","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.709093283Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.709104872Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643,"session_id":"27f33872-93f3-42de-b800-067c2fbc4a72"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:38.70911667Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 149 len 8192 data: 7 7 03/30 Read block 101 len 12288 04/30 Write block 540 len 36864 data: 11 9 6 5 5 9 9 6 8 05/30 Read block 482 len 16384 06/30 Flush 07/30 Write block 90 len 16384 data: 10 9 8 8 08/30 Read block 76 len 36864 09/30 Read block 72 len 40960 10/30 Write block 459 len 12288 data: 6 8 8 11/30 Flush 12/30 Write block 80 len 32768 data: 10 9 9 11 11 10 7 8 13/30 Flush 14/30 Flush 15/30 Flush 16/30 Write block 187 len 40960 data: 9 7 8 9 10 12 10 9 9 10 17/30 Read block 372 len 20480 18/30 Read block 98 len 4096 19/30 Read block 62 len 4096 20/30 Read block 392 len 24576 21/30 Read block 153 len 8192 22/30 Flush 23/30 Write block 317 len 20480 data: 10 8 5 7 9 24/30 Read block 443 len 12288 25/30 Read block 595 len 20480 26/30 Write block 222 len 16384 data: 9 11 12 12 27/30 Write block 515 len 20480 data: 11 14 13 13 15 28/30 Read block 325 len 36864 29/30 Write block 259 len 28672 data: 8 8 10 12 12 10 13 30/30 Write block 147 len 28672 data: 8 6 8 8 5 5 4 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:38.728520181Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1643} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 192 192 192 F F F 1 020-039 105 105 105 191 191 191 F F F 2 040-059 112 112 112 205 205 205 F F F 3 060-079 112 112 112 205 205 205 F F F 4 080-099 114 114 114 211 211 211 F F F 5 100-119 112 112 112 205 205 205 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 114 114 114 209 209 209 T T T 8 160-179 110 110 110 202 202 202 F F F 9 180-199 114 114 114 214 214 214 F F F 10 200-219 111 111 111 203 203 203 F F F 11 220-239 106 106 106 194 194 194 T T T 12 240-259 110 110 110 201 201 201 T T T 13 260-279 104 104 104 190 190 190 T T T 14 280-299 109 109 109 198 198 198 F F F 15 300-319 113 113 113 207 207 207 T T T 16 320-339 113 113 113 207 207 207 T T T 17 340-359 110 110 110 202 202 202 F F F 18 360-379 113 113 113 207 207 207 F F F 19 380-399 112 112 112 205 205 205 F F F 20 400-419 106 106 106 192 192 192 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 114 114 114 210 210 210 F F F 23 460-479 114 114 114 210 210 210 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 113 113 113 207 207 207 T T T 26 520-539 110 110 110 202 202 202 F F F 27 540-559 114 114 114 209 209 209 F F F 28 560-579 113 113 113 207 207 207 F F F 29 580-599 112 112 112 205 205 205 F F F Max gen: 114, Max flush: 214 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 192 192 192 F F F 1 020-039 105 105 105 191 191 191 F F F 2 040-059 112 112 112 205 205 205 F F F 3 060-079 112 112 112 205 205 205 F F F 4 080-099 114 114 114 211 211 211 F F F 5 100-119 112 112 112 205 205 205 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 115 115 115 215 215 215 F F F 8 160-179 110 110 110 202 202 202 F F F 9 180-199 114 114 114 214 214 214 F F F 10 200-219 111 111 111 203 203 203 F F F 11 220-239 115 115 115 215 215 215 F F F 12 240-259 115 115 115 215 215 215 F F F 13 260-279 115 115 115 215 215 215 F F F 14 280-299 109 109 109 198 198 198 F F F 15 300-319 115 115 115 215 215 215 F F F 16 320-339 115 115 115 215 215 215 F F F 17 340-359 110 110 110 202 202 202 F F F 18 360-379 113 113 113 207 207 207 F F F 19 380-399 112 112 112 205 205 205 F F F 20 400-419 106 106 106 192 192 192 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 114 114 114 210 210 210 F F F 23 460-479 114 114 114 210 210 210 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 115 115 115 215 215 215 F F F 26 520-539 110 110 110 202 202 202 F F F 27 540-559 114 114 114 209 209 209 F F F 28 560-579 113 113 113 207 207 207 F F F 29 580-599 112 112 112 205 205 205 F F F Max gen: 115, Max flush: 215 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.005435456Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:43.04720164Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.047580292Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.078582284Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.07924194Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.079275196Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.079288395Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.079302544Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651} {"msg":"Crucible 36bebbcc-c4a1-406b-9e9b-a71726cb1071 has session id: 55597253-e76f-468b-841e-690c3dc8d211","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.079874768Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Upstairs opts: Upstairs UUID: 36bebbcc-c4a1-406b-9e9b-a71726cb1071, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.079921194Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Crucible stats registered with UUID: 36bebbcc-c4a1-406b-9e9b-a71726cb1071","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.079943621Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:43.07995715Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080435284Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080458901Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651} {"msg":"36bebbcc-c4a1-406b-9e9b-a71726cb1071 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080524715Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080542603Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"0","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.0805682Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"0","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080593158Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"1","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080621115Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"1","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080649053Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"2","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.08067643Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"2","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080694098Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080707627Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080733954Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"ds_connection connected from Ok(127.0.0.1:65483)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080760342Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"io task","client":"2","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"ds_connection connected from Ok(127.0.0.1:34140)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080784519Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"io task","client":"1","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"ds_connection connected from Ok(127.0.0.1:47482)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.080935995Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"io task","client":"0","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.081426047Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"1","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.081550225Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"2","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.081758644Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"0","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082058705Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"1","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082247057Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"2","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082394243Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"0","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082649078Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082759177Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"[0]R flush_numbers[0..12]: [192, 191, 205, 205, 211, 205, 176, 215, 202, 214, 203, 215]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082864957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"[0]R generation[0..12]: [106, 105, 112, 112, 114, 112, 98, 115, 110, 114, 111, 115]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082885815Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082904933Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"[1]R flush_numbers[0..12]: [192, 191, 205, 205, 211, 205, 176, 215, 202, 214, 203, 215]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082919591Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"[1]R generation[0..12]: [106, 105, 112, 112, 114, 112, 98, 115, 110, 114, 111, 115]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.08293331Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082947669Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"[2]R flush_numbers[0..12]: [192, 191, 205, 205, 211, 205, 176, 215, 202, 214, 203, 215]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082961977Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"[2]R generation[0..12]: [106, 105, 112, 112, 114, 112, 98, 115, 110, 114, 111, 115]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082975756Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.082989485Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083002334Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083015462Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083028191Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083054368Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"":"downstairs","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083077586Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083090525Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"0","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083108383Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"1","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083131551Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"client":"2","":"downstairs","session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083160988Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083185396Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"36bebbcc-c4a1-406b-9e9b-a71726cb1071 is now active with session: 55597253-e76f-468b-841e-690c3dc8d211","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083210893Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083235041Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651,"session_id":"55597253-e76f-468b-841e-690c3dc8d211"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:43.083259898Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 500 len 28672 data: 9 8 8 10 11 12 11 03/30 Write block 11 len 40960 data: 11 11 11 10 11 11 6 5 7 6 04/30 Write block 23 len 24576 data: 9 8 9 10 9 9 05/30 Read block 595 len 16384 06/30 Write block 191 len 40960 data: 11 13 11 10 10 11 11 9 9 10 07/30 Read block 213 len 28672 08/30 Flush 09/30 Write block 222 len 4096 data: 10 10/30 Read block 247 len 8192 11/30 Write block 107 len 40960 data: 10 9 7 6 5 5 8 9 8 8 12/30 Read block 109 len 8192 13/30 Read block 432 len 40960 14/30 Read block 121 len 16384 15/30 Flush 16/30 Write block 353 len 36864 data: 11 10 9 8 10 12 11 9 8 17/30 Write block 595 len 8192 data: 7 7 18/30 Read block 454 len 24576 19/30 Write block 518 len 40960 data: 14 16 16 13 11 10 10 8 6 4 20/30 Read block 48 len 8192 21/30 Write block 521 len 28672 data: 14 12 11 11 9 7 5 22/30 Write block 503 len 36864 data: 11 12 13 12 9 9 12 13 13 23/30 Read block 461 len 16384 24/30 Write block 539 len 24576 data: 11 12 10 7 6 6 25/30 Read block 417 len 28672 26/30 Write block 253 len 40960 data: 11 10 7 10 8 9 9 9 11 13 27/30 Read block 299 len 4096 28/30 Write block 530 len 24576 data: 11 13 13 12 7 9 29/30 Read block 15 len 16384 30/30 Write block 42 len 20480 data: 8 8 8 6 6 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 9 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:43.104534617Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1651} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 217 217 217 F F F 1 020-039 116 116 116 217 217 217 F F F 2 040-059 112 112 112 205 205 205 T T T 3 060-079 112 112 112 205 205 205 F F F 4 080-099 114 114 114 211 211 211 F F F 5 100-119 116 116 116 218 218 218 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 115 115 115 215 215 215 F F F 8 160-179 110 110 110 202 202 202 F F F 9 180-199 116 116 116 217 217 217 F F F 10 200-219 116 116 116 217 217 217 F F F 11 220-239 116 116 116 218 218 218 F F F 12 240-259 115 115 115 215 215 215 T T T 13 260-279 115 115 115 215 215 215 T T T 14 280-299 109 109 109 198 198 198 F F F 15 300-319 115 115 115 215 215 215 F F F 16 320-339 115 115 115 215 215 215 F F F 17 340-359 110 110 110 202 202 202 T T T 18 360-379 113 113 113 207 207 207 T T T 19 380-399 112 112 112 205 205 205 F F F 20 400-419 106 106 106 192 192 192 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 114 114 114 210 210 210 F F F 23 460-479 114 114 114 210 210 210 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 116 116 116 217 217 217 T T T 26 520-539 110 110 110 202 202 202 T T T 27 540-559 114 114 114 209 209 209 T T T 28 560-579 113 113 113 207 207 207 F F F 29 580-599 112 112 112 205 205 205 T T T Max gen: 116, Max flush: 218 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 217 217 217 F F F 1 020-039 116 116 116 217 217 217 F F F 2 040-059 117 117 117 219 219 219 F F F 3 060-079 112 112 112 205 205 205 F F F 4 080-099 114 114 114 211 211 211 F F F 5 100-119 116 116 116 218 218 218 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 115 115 115 215 215 215 F F F 8 160-179 110 110 110 202 202 202 F F F 9 180-199 116 116 116 217 217 217 F F F 10 200-219 116 116 116 217 217 217 F F F 11 220-239 116 116 116 218 218 218 F F F 12 240-259 117 117 117 219 219 219 F F F 13 260-279 117 117 117 219 219 219 F F F 14 280-299 109 109 109 198 198 198 F F F 15 300-319 115 115 115 215 215 215 F F F 16 320-339 115 115 115 215 215 215 F F F 17 340-359 117 117 117 219 219 219 F F F 18 360-379 117 117 117 219 219 219 F F F 19 380-399 112 112 112 205 205 205 F F F 20 400-419 106 106 106 192 192 192 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 114 114 114 210 210 210 F F F 23 460-479 114 114 114 210 210 210 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 117 117 117 219 219 219 F F F 26 520-539 117 117 117 219 219 219 F F F 27 540-559 117 117 117 219 219 219 F F F 28 560-579 113 113 113 207 207 207 F F F 29 580-599 117 117 117 219 219 219 F F F Max gen: 117, Max flush: 219 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.39204323Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:47.43394567Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.434315714Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.464914844Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.465586339Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.465621335Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.465635344Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.465650392Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659} {"msg":"Crucible 651d1dde-834c-4b0e-92ff-3462dd88cebe has session id: d8220198-c5b7-4923-b7f3-cfa1aa8abe20","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466218607Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Upstairs opts: Upstairs UUID: 651d1dde-834c-4b0e-92ff-3462dd88cebe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466269022Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Crucible stats registered with UUID: 651d1dde-834c-4b0e-92ff-3462dd88cebe","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.46629047Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:47.466304659Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466658824Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466682182Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659} {"msg":"651d1dde-834c-4b0e-92ff-3462dd88cebe active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466742216Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466760724Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"0","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466783982Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"0","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466797891Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"1","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466817089Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"1","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466841696Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"2","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466868514Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"2","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466892482Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466917809Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466943017Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"ds_connection connected from Ok(127.0.0.1:40359)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.466971044Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"io task","client":"2","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"ds_connection connected from Ok(127.0.0.1:62268)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.467030198Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"io task","client":"0","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"ds_connection connected from Ok(127.0.0.1:57018)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.467066994Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"io task","client":"1","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.467685964Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"2","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.467716821Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"0","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.467742649Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"1","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.468351359Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"1","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.468380406Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"0","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.468396635Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"2","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.468955131Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469076199Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"[0]R flush_numbers[0..12]: [217, 217, 219, 205, 211, 218, 176, 215, 202, 217, 217, 218]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469150142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"[0]R generation[0..12]: [116, 116, 117, 112, 114, 116, 98, 115, 110, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.46917098Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469186308Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"[1]R flush_numbers[0..12]: [217, 217, 219, 205, 211, 218, 176, 215, 202, 217, 217, 218]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469200477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"[1]R generation[0..12]: [116, 116, 117, 112, 114, 116, 98, 115, 110, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469214195Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469227894Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"[2]R flush_numbers[0..12]: [217, 217, 219, 205, 211, 218, 176, 215, 202, 217, 217, 218]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469241273Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"[2]R generation[0..12]: [116, 116, 117, 112, 114, 116, 98, 115, 110, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469258541Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.46927273Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Max found gen is 118","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469291078Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469304087Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469316815Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469329504Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"":"downstairs","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469342213Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469354032Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"0","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469367361Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"1","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469380199Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"client":"2","":"downstairs","session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469393088Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469405067Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"651d1dde-834c-4b0e-92ff-3462dd88cebe is now active with session: d8220198-c5b7-4923-b7f3-cfa1aa8abe20","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469417585Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469429364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659,"session_id":"d8220198-c5b7-4923-b7f3-cfa1aa8abe20"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:47.469441373Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 254 len 32768 data: 11 8 11 9 10 10 10 12 02/30 Read block 325 len 32768 03/30 Read block 578 len 4096 04/30 Write block 177 len 32768 data: 13 12 14 16 12 12 13 11 05/30 Read block 544 len 28672 06/30 Write block 105 len 24576 data: 8 8 11 10 8 7 07/30 Read block 102 len 24576 08/30 Write block 286 len 20480 data: 6 7 7 9 9 09/30 Write block 307 len 20480 data: 11 12 15 16 14 10/30 Write block 445 len 8192 data: 15 14 11/30 Read block 261 len 4096 12/30 Read block 589 len 4096 13/30 Write block 82 len 36864 data: 10 12 12 11 8 9 8 10 11 14/30 Write block 585 len 8192 data: 17 11 15/30 Read block 513 len 16384 16/30 Write block 253 len 28672 data: 12 12 9 12 10 11 11 17/30 Write block 410 len 32768 data: 9 7 7 7 7 7 9 8 18/30 Read block 583 len 28672 19/30 Read block 380 len 40960 20/30 Write block 400 len 40960 data: 9 9 8 12 10 10 10 9 11 9 21/30 Read block 379 len 12288 22/30 Write block 237 len 12288 data: 13 15 14 23/30 Read block 342 len 8192 24/30 Write block 400 len 40960 data: 10 10 9 13 11 11 11 10 12 10 25/30 Write block 193 len 12288 data: 12 11 11 26/30 Read block 370 len 32768 27/30 Write block 112 len 16384 data: 6 9 10 9 28/30 Write block 5 len 4096 data: 5 29/30 Read block 498 len 12288 30/30 Read block 137 len 16384 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Write 2 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:47.488563351Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1659} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 217 217 217 T T T 1 020-039 116 116 116 217 217 217 F F F 2 040-059 117 117 117 219 219 219 F F F 3 060-079 112 112 112 205 205 205 F F F 4 080-099 114 114 114 211 211 211 T T T 5 100-119 116 116 116 218 218 218 T T T 6 120-139 98 98 98 176 176 176 F F F 7 140-159 115 115 115 215 215 215 F F F 8 160-179 110 110 110 202 202 202 T T T 9 180-199 116 116 116 217 217 217 T T T 10 200-219 116 116 116 217 217 217 F F F 11 220-239 116 116 116 218 218 218 T T T 12 240-259 117 117 117 219 219 219 T T T 13 260-279 117 117 117 219 219 219 T T T 14 280-299 109 109 109 198 198 198 T T T 15 300-319 115 115 115 215 215 215 T T T 16 320-339 115 115 115 215 215 215 F F F 17 340-359 117 117 117 219 219 219 F F F 18 360-379 117 117 117 219 219 219 F F F 19 380-399 112 112 112 205 205 205 F F F 20 400-419 106 106 106 192 192 192 T T T 21 420-439 109 109 109 198 198 198 F F F 22 440-459 114 114 114 210 210 210 T T T 23 460-479 114 114 114 210 210 210 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 117 117 117 219 219 219 F F F 26 520-539 117 117 117 219 219 219 F F F 27 540-559 117 117 117 219 219 219 F F F 28 560-579 113 113 113 207 207 207 F F F 29 580-599 117 117 117 219 219 219 T T T Max gen: 117, Max flush: 219 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 220 220 220 F F F 1 020-039 116 116 116 217 217 217 F F F 2 040-059 117 117 117 219 219 219 F F F 3 060-079 112 112 112 205 205 205 F F F 4 080-099 118 118 118 220 220 220 F F F 5 100-119 118 118 118 220 220 220 F F F 6 120-139 98 98 98 176 176 176 F F F 7 140-159 115 115 115 215 215 215 F F F 8 160-179 118 118 118 220 220 220 F F F 9 180-199 118 118 118 220 220 220 F F F 10 200-219 116 116 116 217 217 217 F F F 11 220-239 118 118 118 220 220 220 F F F 12 240-259 118 118 118 220 220 220 F F F 13 260-279 118 118 118 220 220 220 F F F 14 280-299 118 118 118 220 220 220 F F F 15 300-319 118 118 118 220 220 220 F F F 16 320-339 115 115 115 215 215 215 F F F 17 340-359 117 117 117 219 219 219 F F F 18 360-379 117 117 117 219 219 219 F F F 19 380-399 112 112 112 205 205 205 F F F 20 400-419 118 118 118 220 220 220 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 118 118 118 220 220 220 F F F 23 460-479 114 114 114 210 210 210 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 117 117 117 219 219 219 F F F 26 520-539 117 117 117 219 219 219 F F F 27 540-559 117 117 117 219 219 219 F F F 28 560-579 113 113 113 207 207 207 F F F 29 580-599 118 118 118 220 220 220 F F F Max gen: 118, Max flush: 220 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.793762793Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.837159487Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.837805664Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.8378513Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.837866248Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.837880797Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667} {"msg":"Crucible 9f862aff-ed18-4584-8b00-14d7aa2bf40c has session id: 5fbfa398-2601-46de-9950-078f8fbb4c8c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.83846707Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Upstairs opts: Upstairs UUID: 9f862aff-ed18-4584-8b00-14d7aa2bf40c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.838516735Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Crucible stats registered with UUID: 9f862aff-ed18-4584-8b00-14d7aa2bf40c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.838543742Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:51.83856635Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839057152Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.83908278Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667} {"msg":"9f862aff-ed18-4584-8b00-14d7aa2bf40c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839135715Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839162372Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"0","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.83918682Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"0","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839201048Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"1","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839215477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"1","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839232945Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"2","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839255203Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"2","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839278211Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839301509Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839325026Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52692)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839503019Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"io task","client":"2","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61071)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839543065Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"io task","client":"0","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39592)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.839572222Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"io task","client":"1","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.840152816Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"2","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.840180813Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"0","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.84031012Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"1","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.840782724Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"0","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.840812511Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"2","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.840914161Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"1","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841375397Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841475277Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"[0]R flush_numbers[0..12]: [220, 217, 219, 205, 220, 220, 176, 215, 220, 220, 217, 220]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841541131Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"[0]R generation[0..12]: [118, 116, 117, 112, 118, 118, 98, 115, 118, 118, 116, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841560229Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841574587Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"[1]R flush_numbers[0..12]: [220, 217, 219, 205, 220, 220, 176, 215, 220, 220, 217, 220]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841588216Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"[1]R generation[0..12]: [118, 116, 117, 112, 118, 118, 98, 115, 118, 118, 116, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841601575Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841615303Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"[2]R flush_numbers[0..12]: [220, 217, 219, 205, 220, 220, 176, 215, 220, 220, 217, 220]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841628952Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"[2]R generation[0..12]: [118, 116, 117, 112, 118, 118, 98, 115, 118, 118, 116, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841642301Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841655779Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Max found gen is 119","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841668578Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Generation requested: 120 >= found:119","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841681517Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841694056Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841710164Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841729982Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841741781Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"0","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.84175479Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"1","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841767709Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"client":"2","":"downstairs","session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841780557Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841792506Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"9f862aff-ed18-4584-8b00-14d7aa2bf40c is now active with session: 5fbfa398-2601-46de-9950-078f8fbb4c8c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841805045Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841816734Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667,"session_id":"5fbfa398-2601-46de-9950-078f8fbb4c8c"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:51.841828632Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 14 len 28672 data: 11 12 12 7 6 8 7 02/30 Write block 444 len 32768 data: 14 16 15 15 13 10 12 9 03/30 Read block 456 len 4096 04/30 Write block 333 len 40960 data: 10 10 7 8 8 7 9 8 8 11 05/30 Read block 379 len 36864 06/30 Write block 99 len 4096 data: 6 07/30 Write block 120 len 20480 data: 6 6 5 5 4 08/30 Read block 136 len 32768 09/30 Read block 300 len 12288 10/30 Read block 196 len 4096 11/30 Flush 12/30 Write block 382 len 24576 data: 8 9 10 9 10 10 13/30 Write block 456 len 32768 data: 8 7 6 7 9 9 11 10 14/30 Write block 67 len 24576 data: 12 12 13 10 13 14 15/30 Flush 16/30 Write block 262 len 4096 data: 14 17/30 Write block 399 len 32768 data: 9 11 11 10 14 12 12 12 18/30 Read block 112 len 8192 19/30 Write block 208 len 24576 data: 10 10 12 10 9 11 20/30 Read block 582 len 12288 21/30 Read block 254 len 12288 22/30 Write block 407 len 16384 data: 11 13 11 10 23/30 Read block 417 len 28672 24/30 Read block 112 len 8192 25/30 Read block 260 len 36864 26/30 Write block 119 len 4096 data: 6 27/30 Write block 198 len 12288 data: 10 10 11 28/30 Write block 222 len 40960 data: 11 12 13 13 12 10 11 13 9 7 29/30 Write block 119 len 16384 data: 7 7 7 6 30/30 Write block 454 len 36864 data: 7 8 9 8 7 8 10 10 12 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 1 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:51.860613043Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1667} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 221 221 221 F F F 1 020-039 120 120 120 221 221 221 F F F 2 040-059 117 117 117 219 219 219 F F F 3 060-079 120 120 120 222 222 222 F F F 4 080-099 120 120 120 221 221 221 F F F 5 100-119 118 118 118 220 220 220 T T T 6 120-139 120 120 120 221 221 221 T T T 7 140-159 115 115 115 215 215 215 F F F 8 160-179 118 118 118 220 220 220 F F F 9 180-199 118 118 118 220 220 220 T T T 10 200-219 116 116 116 217 217 217 T T T 11 220-239 118 118 118 220 220 220 T T T 12 240-259 118 118 118 220 220 220 F F F 13 260-279 118 118 118 220 220 220 T T T 14 280-299 118 118 118 220 220 220 F F F 15 300-319 118 118 118 220 220 220 F F F 16 320-339 120 120 120 221 221 221 F F F 17 340-359 120 120 120 221 221 221 F F F 18 360-379 117 117 117 219 219 219 F F F 19 380-399 120 120 120 222 222 222 T T T 20 400-419 118 118 118 220 220 220 T T T 21 420-439 109 109 109 198 198 198 F F F 22 440-459 120 120 120 222 222 222 T T T 23 460-479 120 120 120 222 222 222 T T T 24 480-499 112 112 112 206 206 206 F F F 25 500-519 117 117 117 219 219 219 F F F 26 520-539 117 117 117 219 219 219 F F F 27 540-559 117 117 117 219 219 219 F F F 28 560-579 113 113 113 207 207 207 F F F 29 580-599 118 118 118 220 220 220 F F F Max gen: 120, Max flush: 222 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 221 221 221 F F F 1 020-039 120 120 120 221 221 221 F F F 2 040-059 117 117 117 219 219 219 F F F 3 060-079 120 120 120 222 222 222 F F F 4 080-099 120 120 120 221 221 221 F F F 5 100-119 121 121 121 223 223 223 F F F 6 120-139 121 121 121 223 223 223 F F F 7 140-159 115 115 115 215 215 215 F F F 8 160-179 118 118 118 220 220 220 F F F 9 180-199 121 121 121 223 223 223 F F F 10 200-219 121 121 121 223 223 223 F F F 11 220-239 121 121 121 223 223 223 F F F 12 240-259 118 118 118 220 220 220 F F F 13 260-279 121 121 121 223 223 223 F F F 14 280-299 118 118 118 220 220 220 F F F 15 300-319 118 118 118 220 220 220 F F F 16 320-339 120 120 120 221 221 221 F F F 17 340-359 120 120 120 221 221 221 F F F 18 360-379 117 117 117 219 219 219 F F F 19 380-399 121 121 121 223 223 223 F F F 20 400-419 121 121 121 223 223 223 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 121 121 121 223 223 223 F F F 23 460-479 121 121 121 223 223 223 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 117 117 117 219 219 219 F F F 26 520-539 117 117 117 219 219 219 F F F 27 540-559 117 117 117 219 219 219 F F F 28 560-579 113 113 113 207 207 207 F F F 29 580-599 118 118 118 220 220 220 F F F Max gen: 121, Max flush: 223 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.121614808Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:56.163553984Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.163975473Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.19492234Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.195486475Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.195526311Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.195542769Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.195559868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675} {"msg":"Crucible 88babe55-232b-470e-b508-144187eea9dc has session id: e8219170-ce13-4e03-a7de-f9b83ae00a43","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.19615527Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Upstairs opts: Upstairs UUID: 88babe55-232b-470e-b508-144187eea9dc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196195676Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Crucible stats registered with UUID: 88babe55-232b-470e-b508-144187eea9dc","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196215314Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:56.196228672Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196590737Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196615185Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675} {"msg":"88babe55-232b-470e-b508-144187eea9dc active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196691407Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196708966Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"0","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196730844Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"0","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196744612Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"1","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196761341Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"1","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196785588Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"2","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196812326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"2","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196838083Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196863821Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196891748Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"ds_connection connected from Ok(127.0.0.1:65304)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196908406Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"ds_connection connected from Ok(127.0.0.1:55622)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.196940703Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"ds_connection connected from Ok(127.0.0.1:63240)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.197123785Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.197679071Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"0","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.197706769Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"2","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.197722387Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"1","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.198220589Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"1","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.19831408Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"0","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.198530119Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"2","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.198852737Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.198940708Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"[0]R flush_numbers[0..12]: [221, 221, 219, 222, 221, 223, 223, 215, 220, 223, 223, 223]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199050628Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"[0]R generation[0..12]: [120, 120, 117, 120, 120, 121, 121, 115, 118, 121, 121, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199072126Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199086694Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"[1]R flush_numbers[0..12]: [221, 221, 219, 222, 221, 223, 223, 215, 220, 223, 223, 223]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199100323Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"[1]R generation[0..12]: [120, 120, 117, 120, 120, 121, 121, 115, 118, 121, 121, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199117561Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.19913279Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"[2]R flush_numbers[0..12]: [221, 221, 219, 222, 221, 223, 223, 215, 220, 223, 223, 223]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199146519Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"[2]R generation[0..12]: [120, 120, 117, 120, 120, 121, 121, 115, 118, 121, 121, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199159747Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199173156Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199185615Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199208213Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.19923183Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199257348Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"":"downstairs","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199282875Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199307593Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"0","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.19933392Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"1","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199360938Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"client":"2","":"downstairs","session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199388015Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199416712Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"88babe55-232b-470e-b508-144187eea9dc is now active with session: e8219170-ce13-4e03-a7de-f9b83ae00a43","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199446809Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199471077Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675,"session_id":"e8219170-ce13-4e03-a7de-f9b83ae00a43"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:13:56.199484415Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 313 len 32768 data: 13 12 14 13 11 9 6 8 02/30 Write block 516 len 24576 data: 15 14 15 17 17 15 03/30 Read block 367 len 16384 04/30 Write block 544 len 24576 data: 7 10 10 7 9 10 05/30 Read block 80 len 16384 06/30 Read block 205 len 32768 07/30 Flush 08/30 Write block 337 len 40960 data: 9 8 10 9 9 12 11 11 11 13 09/30 Read block 531 len 28672 10/30 Read block 441 len 24576 11/30 Write block 461 len 32768 data: 11 13 11 11 13 10 9 6 12/30 Write block 365 len 8192 data: 7 6 13/30 Flush 14/30 Write block 312 len 4096 data: 12 15/30 Write block 577 len 4096 data: 11 16/30 Read block 484 len 16384 17/30 Write block 202 len 40960 data: 9 11 12 11 10 10 11 11 13 11 18/30 Write block 262 len 36864 data: 15 13 11 14 14 13 11 9 9 19/30 Write block 199 len 36864 data: 11 12 9 10 12 13 12 11 11 20/30 Write block 459 len 24576 data: 9 11 12 14 12 12 21/30 Read block 364 len 28672 22/30 Flush 23/30 Write block 294 len 40960 data: 9 7 9 9 9 8 12 13 10 11 24/30 Read block 126 len 32768 25/30 Write block 309 len 4096 data: 16 26/30 Write block 65 len 4096 data: 12 27/30 Write block 261 len 28672 data: 13 16 14 12 15 15 14 28/30 Read block 344 len 4096 29/30 Write block 278 len 28672 data: 11 10 7 6 7 5 4 30/30 Read block 11 len 36864 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:13:56.219403316Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1675} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 221 221 221 F F F 1 020-039 120 120 120 221 221 221 F F F 2 040-059 117 117 117 219 219 219 F F F 3 060-079 120 120 120 222 222 222 T T T 4 080-099 120 120 120 221 221 221 F F F 5 100-119 121 121 121 223 223 223 F F F 6 120-139 121 121 121 223 223 223 F F F 7 140-159 115 115 115 215 215 215 F F F 8 160-179 118 118 118 220 220 220 F F F 9 180-199 122 122 122 226 226 226 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 121 121 121 223 223 223 F F F 12 240-259 118 118 118 220 220 220 F F F 13 260-279 122 122 122 226 226 226 T T T 14 280-299 118 118 118 220 220 220 T T T 15 300-319 122 122 122 226 226 226 T T T 16 320-339 122 122 122 225 225 225 F F F 17 340-359 122 122 122 225 225 225 F F F 18 360-379 122 122 122 225 225 225 F F F 19 380-399 121 121 121 223 223 223 F F F 20 400-419 121 121 121 223 223 223 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 122 122 122 226 226 226 F F F 23 460-479 122 122 122 226 226 226 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 122 122 122 224 224 224 F F F 26 520-539 122 122 122 224 224 224 F F F 27 540-559 122 122 122 224 224 224 F F F 28 560-579 122 122 122 226 226 226 F F F 29 580-599 118 118 118 220 220 220 F F F Max gen: 122, Max flush: 226 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 221 221 221 F F F 1 020-039 120 120 120 221 221 221 F F F 2 040-059 117 117 117 219 219 219 F F F 3 060-079 123 123 123 227 227 227 F F F 4 080-099 120 120 120 221 221 221 F F F 5 100-119 121 121 121 223 223 223 F F F 6 120-139 121 121 121 223 223 223 F F F 7 140-159 115 115 115 215 215 215 F F F 8 160-179 118 118 118 220 220 220 F F F 9 180-199 122 122 122 226 226 226 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 121 121 121 223 223 223 F F F 12 240-259 118 118 118 220 220 220 F F F 13 260-279 123 123 123 227 227 227 F F F 14 280-299 123 123 123 227 227 227 F F F 15 300-319 123 123 123 227 227 227 F F F 16 320-339 122 122 122 225 225 225 F F F 17 340-359 122 122 122 225 225 225 F F F 18 360-379 122 122 122 225 225 225 F F F 19 380-399 121 121 121 223 223 223 F F F 20 400-419 121 121 121 223 223 223 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 122 122 122 226 226 226 F F F 23 460-479 122 122 122 226 226 226 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 122 122 122 224 224 224 F F F 26 520-539 122 122 122 224 224 224 F F F 27 540-559 122 122 122 224 224 224 F F F 28 560-579 122 122 122 226 226 226 F F F 29 580-599 118 118 118 220 220 220 F F F Max gen: 123, Max flush: 227 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.476341636Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.519504883Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.520092986Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.520135892Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.52015158Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.520168369Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683} {"msg":"Crucible 343614c0-9225-4c96-9f07-1132fca63b8a has session id: 8f5a5515-1921-4cd1-8913-99cffb53483c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.520764571Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Upstairs opts: Upstairs UUID: 343614c0-9225-4c96-9f07-1132fca63b8a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.520807507Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Crucible stats registered with UUID: 343614c0-9225-4c96-9f07-1132fca63b8a","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.520830474Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:00.520850742Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521361903Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.52138906Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683} {"msg":"343614c0-9225-4c96-9f07-1132fca63b8a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521466603Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521484971Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"0","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521508278Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"0","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521522087Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"1","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521536056Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"1","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521554584Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"2","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521581241Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"2","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521606019Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521632536Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521660344Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43427)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521688641Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35372)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521715588Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33014)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.521913539Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.522379334Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"2","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.522539298Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"0","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.52292612Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"2","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.523064177Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"1","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.523115942Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"0","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.523519692Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.523663598Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"1","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.523687256Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"[0]R flush_numbers[0..12]: [221, 221, 219, 227, 221, 223, 223, 215, 220, 226, 226, 223]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524178868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"[0]R generation[0..12]: [120, 120, 117, 123, 120, 121, 121, 115, 118, 122, 122, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524206246Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524221274Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"[1]R flush_numbers[0..12]: [221, 221, 219, 227, 221, 223, 223, 215, 220, 226, 226, 223]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524235683Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"[1]R generation[0..12]: [120, 120, 117, 123, 120, 121, 121, 115, 118, 122, 122, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524249652Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.52426379Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"[2]R flush_numbers[0..12]: [221, 221, 219, 227, 221, 223, 223, 215, 220, 226, 226, 223]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524277409Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"[2]R generation[0..12]: [120, 120, 117, 123, 120, 121, 121, 115, 118, 122, 122, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524290817Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524304666Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524317345Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524330194Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Next flush: 228","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:14:00.524342712Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.52436337Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"":"downstairs","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524376729Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524388558Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"0","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524401557Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"1","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524414335Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"client":"2","":"downstairs","session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524427274Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524439203Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"343614c0-9225-4c96-9f07-1132fca63b8a is now active with session: 8f5a5515-1921-4cd1-8913-99cffb53483c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524451702Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524463361Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683,"session_id":"8f5a5515-1921-4cd1-8913-99cffb53483c"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:00.524475439Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 408 len 16384 data: 14 12 11 8 02/30 Read block 254 len 16384 03/30 Read block 116 len 16384 04/30 Read block 191 len 28672 05/30 Write block 93 len 36864 data: 9 6 7 8 9 8 7 6 6 06/30 Read block 523 len 36864 07/30 Write block 33 len 36864 data: 8 9 8 7 8 9 8 9 8 08/30 Read block 245 len 32768 09/30 Write block 324 len 28672 data: 14 14 13 14 15 14 12 10/30 Write block 356 len 8192 data: 9 11 11/30 Read block 550 len 28672 12/30 Write block 178 len 8192 data: 13 15 13/30 Read block 78 len 4096 14/30 Write block 133 len 12288 data: 9 7 8 15/30 Write block 84 len 24576 data: 13 12 9 10 9 11 16/30 Flush 17/30 Write block 515 len 16384 data: 12 16 15 16 18/30 Read block 371 len 12288 19/30 Read block 433 len 24576 20/30 Read block 290 len 8192 21/30 Write block 588 len 24576 data: 10 13 11 11 11 11 22/30 Read block 37 len 16384 23/30 Read block 237 len 28672 24/30 Write block 244 len 28672 data: 11 10 9 11 13 10 12 25/30 Write block 247 len 4096 data: 12 26/30 Read block 584 len 24576 27/30 Read block 200 len 32768 28/30 Read block 120 len 28672 29/30 Write block 382 len 12288 data: 9 10 11 30/30 Read block 164 len 8192 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 4 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 14 12 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:00.544478422Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1683} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 221 221 221 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 124 124 124 228 228 228 F F F 3 060-079 123 123 123 227 227 227 F F F 4 080-099 124 124 124 228 228 228 F F F 5 100-119 124 124 124 228 228 228 F F F 6 120-139 124 124 124 228 228 228 F F F 7 140-159 115 115 115 215 215 215 F F F 8 160-179 124 124 124 228 228 228 F F F 9 180-199 122 122 122 226 226 226 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 121 121 121 223 223 223 F F F 12 240-259 118 118 118 220 220 220 T T T 13 260-279 123 123 123 227 227 227 F F F 14 280-299 123 123 123 227 227 227 F F F 15 300-319 123 123 123 227 227 227 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 124 124 124 228 228 228 F F F 18 360-379 122 122 122 225 225 225 F F F 19 380-399 121 121 121 223 223 223 T T T 20 400-419 124 124 124 228 228 228 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 122 122 122 226 226 226 F F F 23 460-479 122 122 122 226 226 226 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 122 122 122 224 224 224 T T T 26 520-539 122 122 122 224 224 224 F F F 27 540-559 122 122 122 224 224 224 F F F 28 560-579 122 122 122 226 226 226 F F F 29 580-599 118 118 118 220 220 220 T T T Max gen: 124, Max flush: 228 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 221 221 221 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 124 124 124 228 228 228 F F F 3 060-079 123 123 123 227 227 227 F F F 4 080-099 124 124 124 228 228 228 F F F 5 100-119 124 124 124 228 228 228 F F F 6 120-139 124 124 124 228 228 228 F F F 7 140-159 115 115 115 215 215 215 F F F 8 160-179 124 124 124 228 228 228 F F F 9 180-199 122 122 122 226 226 226 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 121 121 121 223 223 223 F F F 12 240-259 125 125 125 229 229 229 F F F 13 260-279 123 123 123 227 227 227 F F F 14 280-299 123 123 123 227 227 227 F F F 15 300-319 123 123 123 227 227 227 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 124 124 124 228 228 228 F F F 18 360-379 122 122 122 225 225 225 F F F 19 380-399 125 125 125 229 229 229 F F F 20 400-419 124 124 124 228 228 228 F F F 21 420-439 109 109 109 198 198 198 F F F 22 440-459 122 122 122 226 226 226 F F F 23 460-479 122 122 122 226 226 226 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 125 125 125 229 229 229 F F F 26 520-539 122 122 122 224 224 224 F F F 27 540-559 122 122 122 224 224 224 F F F 28 560-579 122 122 122 226 226 226 F F F 29 580-599 125 125 125 229 229 229 F F F Max gen: 125, Max flush: 229 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.773328977Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.816483125Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.817057649Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.817099804Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.817114813Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.817129771Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691} {"msg":"Crucible 187a8dc1-2fa9-4066-8c28-44e39763bc3d has session id: 314a8302-7188-4b45-aa26-69d736c10f8a","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.817638412Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Upstairs opts: Upstairs UUID: 187a8dc1-2fa9-4066-8c28-44e39763bc3d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.817683817Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Crucible stats registered with UUID: 187a8dc1-2fa9-4066-8c28-44e39763bc3d","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.817707625Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:04.817724913Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818115735Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818258612Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691} {"msg":"187a8dc1-2fa9-4066-8c28-44e39763bc3d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818287889Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818310956Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"0","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818343293Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"0","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818359312Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"1","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.81837391Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"1","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818390049Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"2","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818404277Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"2","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818473761Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818502908Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818525246Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51843)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818549613Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:42675)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818576171Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64484)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.818611337Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.819158984Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"1","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.819187881Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"0","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.819484862Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"2","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.819788103Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"0","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.81991945Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"1","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820231589Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"2","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.82043262Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820549958Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"[0]R flush_numbers[0..12]: [221, 228, 228, 227, 228, 228, 228, 215, 228, 226, 226, 223]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820729311Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"[0]R generation[0..12]: [120, 124, 124, 123, 124, 124, 124, 115, 124, 122, 122, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820748139Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820762328Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"[1]R flush_numbers[0..12]: [221, 228, 228, 227, 228, 228, 228, 215, 228, 226, 226, 223]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820775996Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"[1]R generation[0..12]: [120, 124, 124, 123, 124, 124, 124, 115, 124, 122, 122, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820789595Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820803144Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"[2]R flush_numbers[0..12]: [221, 228, 228, 227, 228, 228, 228, 215, 228, 226, 226, 223]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820820872Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"[2]R generation[0..12]: [120, 124, 124, 123, 124, 124, 124, 115, 124, 122, 122, 121]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820835001Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820848649Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Max found gen is 126","v":0,"name"Wait for a query_work_queue command to finish before sending IO :"crucible","level":30,"time":"2025-01-28T17:14:04.820868167Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820881256Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Next flush: 230","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820893885Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820906554Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"":"downstairs","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820919522Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820931321Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"0","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.82094455Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"1","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820957339Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"client":"2","":"downstairs","session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820970208Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820982276Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"187a8dc1-2fa9-4066-8c28-44e39763bc3d is now active with session: 314a8302-7188-4b45-aa26-69d736c10f8a","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.820994885Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.821006624Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691,"session_id":"314a8302-7188-4b45-aa26-69d736c10f8a"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:04.821020423Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 239 len 28672 data: 15 15 14 14 12 12 11 02/30 Read block 343 len 16384 03/30 Write block 91 len 40960 data: 10 9 10 7 8 9 10 9 8 7 04/30 Read block 163 len 28672 05/30 Read block 391 len 8192 06/30 Read block 293 len 28672 07/30 Read block 561 len 4096 08/30 Read block 62 len 8192 09/30 Write block 131 len 16384 data: 8 9 10 8 10/30 Flush 11/30 Write block 396 len 12288 data: 8 7 8 12/30 Write block 410 len 8192 data: 12 9 13/30 Write block 242 len 4096 data: 15 14/30 Read block 16 len 20480 15/30 Read block 333 len 16384 16/30 Read block 290 len 24576 17/30 Write block 95 len 12288 data: 9 10 11 18/30 Write block 289 len 16384 data: 10 10 9 9 19/30 Read block 1 len 36864 20/30 Write block 486 len 16384 data: 9 13 11 11 21/30 Write block 358 len 8192 data: 13 12 22/30 Write block 9 len 4096 data: 9 23/30 Read block 416 len 8192 24/30 Write block 381 len 20480 data: 8 10 11 12 10 25/30 Read block 467 len 16384 26/30 Write block 428 len 32768 data: 9 9 10 10 9 12 14 13 27/30 Write block 158 len 36864 data: 11 10 10 11 12 14 17 13 11 28/30 Write block 555 len 16384 data: 16 14 14 13 29/30 Read block 131 len 28672 30/30 Read block 12 len 24576 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 3 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 19 20 19 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:04.839172215Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1691} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 221 221 221 T T T 1 020-039 124 124 124 228 228 228 F F F 2 040-059 124 124 124 228 228 228 F F F 3 060-079 123 123 123 227 227 227 F F F 4 080-099 126 126 126 230 230 230 T T T 5 100-119 126 126 126 230 230 230 F F F 6 120-139 126 126 126 230 230 230 F F F 7 140-159 115 115 115 215 215 215 T T T 8 160-179 124 124 124 228 228 228 T T T 9 180-199 122 122 122 226 226 226 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 126 126 126 230 230 230 F F F 12 240-259 126 126 126 230 230 230 T T T 13 260-279 123 123 123 227 227 227 F F F 14 280-299 123 123 123 227 227 227 T T T 15 300-319 123 123 123 227 227 227 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 124 124 124 228 228 228 T T T 18 360-379 122 122 122 225 225 225 F F F 19 380-399 125 125 125 229 229 229 T T T 20 400-419 124 124 124 228 228 228 T T T 21 420-439 109 109 109 198 198 198 T T T 22 440-459 122 122 122 226 226 226 F F F 23 460-479 122 122 122 226 226 226 F F F 24 480-499 112 112 112 206 206 206 T T T 25 500-519 125 125 125 229 229 229 F F F 26 520-539 122 122 122 224 224 224 F F F 27 540-559 122 122 122 224 224 224 T T T 28 560-579 122 122 122 226 226 226 F F F 29 580-599 125 125 125 229 229 229 F F F Max gen: 126, Max flush: 230 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 231 231 231 F F F 1 020-039 124 124 124 228 228 228 F F F 2 040-059 124 124 124 228 228 228 F F F 3 060-079 123 123 123 227 227 227 F F F 4 080-099 127 127 127 231 231 231 F F F 5 100-119 126 126 126 230 230 230 F F F 6 120-139 126 126 126 230 230 230 F F F 7 140-159 127 127 127 231 231 231 F F F 8 160-179 127 127 127 231 231 231 F F F 9 180-199 122 122 122 226 226 226 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 126 126 126 230 230 230 F F F 12 240-259 127 127 127 231 231 231 F F F 13 260-279 123 123 123 227 227 227 F F F 14 280-299 127 127 127 231 231 231 F F F 15 300-319 123 123 123 227 227 227 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 127 127 127 231 231 231 F F F 18 360-379 122 122 122 225 225 225 F F F 19 380-399 127 127 127 231 231 231 F F F 20 400-419 127 127 127 231 231 231 F F F 21 420-439 127 127 127 231 231 231 F F F 22 440-459 122 122 122 226 226 226 F F F 23 460-479 122 122 122 226 226 226 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 125 125 125 229 229 229 F F F 26 520-539 122 122 122 224 224 224 F F F 27 540-559 127 127 127 231 231 231 F F F 28 560-579 122 122 122 226 226 226 F F F 29 580-599 125 125 125 229 229 229 F F F Max gen: 127, Max flush: 231 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.114866469Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.157830675Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.158451655Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.158491981Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.158511079Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.158525917Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699} {"msg":"Crucible 554249d1-b817-4d35-ad16-09d3f9ce592c has session id: 9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.159136638Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Upstairs opts: Upstairs UUID: 554249d1-b817-4d35-ad16-09d3f9ce592c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.159236988Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Crucible stats registered with UUID: 554249d1-b817-4d35-ad16-09d3f9ce592c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.159257316Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:09.159270165Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.159845639Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.159876776Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699} {"msg":"554249d1-b817-4d35-ad16-09d3f9ce592c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.159907753Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.159921912Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"0","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.159944219Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"0","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.159957998Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"1","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.159971937Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"1","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.159992915Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"2","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.160021452Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"2","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.160047839Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.160072797Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.160098224Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"ds_connection connected from Ok(127.0.0.1:47167)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.160125792Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"ds_connection connected from Ok(127.0.0.1:42660)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.160152869Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"ds_connection connected from Ok(127.0.0.1:61178)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.160180736Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.160803006Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"0","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.160842872Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"2","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.160950592Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"1","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.161435594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"0","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.161464071Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"1","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.16147889Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"2","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162012748Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162038375Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"[0]R flush_numbers[0..12]: [231, 228, 228, 227, 231, 230, 230, 231, 231, 226, 226, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162066923Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"[0]R generation[0..12]: [127, 124, 124, 123, 127, 126, 126, 127, 127, 122, 122, 126]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162081971Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162107189Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"[1]R flush_numbers[0..12]: [231, 228, 228, 227, 231, 230, 230, 231, 231, 226, 226, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162121227Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"[1]R generation[0..12]: [127, 124, 124, 123, 127, 126, 126, 127, 127, 122, 122, 126]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162134706Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162148415Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"[2]R flush_numbers[0..12]: [231, 228, 228, 227, 231, 230, 230, 231, 231, 226, 226, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162161854Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"[2]R generation[0..12]: [127, 124, 124, 123, 127, 126, 126, 127, 127, 122, 122, 126]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162175092Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162188701Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.16220124Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162214209Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162226967Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162239476Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"":"downstairs","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162260984Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162273353Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"0","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162286342Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"1","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.16229905Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"client":"2","":"downstairs","session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162311979Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162323918Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"554249d1-b817-4d35-ad16-09d3f9ce592c is now active with session: 9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162336417Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162348056Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699,"session_id":"9b5727b0-f1b7-49e9-aadc-a51c7e4a2b88"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:09.162359974Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 349 len 28672 data: 12 11 11 10 12 11 10 02/30 Write block 566 len 40960 data: 6 7 10 11 13 13 12 12 10 8 03/30 Read block 193 len 28672 04/30 Write block 61 len 4096 data: 10 05/30 Flush 06/30 Write block 13 len 36864 data: 12 12 13 13 8 7 9 8 6 07/30 Read block 437 len 32768 08/30 Read block 496 len 20480 09/30 Read block 204 len 40960 10/30 Read block 128 len 16384 11/30 Write block 88 len 28672 data: 10 12 12 11 10 11 8 12/30 Write block 564 len 24576 data: 6 6 7 8 11 12 13/30 Write block 235 len 4096 data: 10 14/30 Read block 45 len 4096 15/30 Read block 573 len 8192 16/30 Write block 577 len 32768 data: 12 14 11 12 13 16 17 18 17/30 Write block 3 len 28672 data: 3 4 6 5 8 9 10 18/30 Write block 425 len 40960 data: 7 9 9 10 10 11 11 10 13 15 19/30 Read block 188 len 40960 20/30 Read block 277 len 4096 21/30 Read block 61 len 36864 22/30 Read block 220 len 8192 23/30 Read block 363 len 32768 24/30 Flush 25/30 Write block 524 len 32768 data: 12 10 8 6 4 8 12 14 26/30 Flush 27/30 Write block 112 len 28672 data: 7 10 11 10 9 8 6 28/30 Read block 32 len 16384 29/30 Read block 555 len 8192 30/30 Read block 101 len 24576 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 4 3 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:09.182752958Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1699} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 233 233 233 F F F 1 020-039 128 128 128 233 233 233 F F F 2 040-059 124 124 124 228 228 228 F F F 3 060-079 128 128 128 232 232 232 F F F 4 080-099 128 128 128 233 233 233 F F F 5 100-119 126 126 126 230 230 230 T T T 6 120-139 126 126 126 230 230 230 F F F 7 140-159 127 127 127 231 231 231 F F F 8 160-179 127 127 127 231 231 231 F F F 9 180-199 122 122 122 226 226 226 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 128 128 128 233 233 233 F F F 12 240-259 127 127 127 231 231 231 F F F 13 260-279 123 123 123 227 227 227 F F F 14 280-299 127 127 127 231 231 231 F F F 15 300-319 123 123 123 227 227 227 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 128 128 128 232 232 232 F F F 18 360-379 122 122 122 225 225 225 F F F 19 380-399 127 127 127 231 231 231 F F F 20 400-419 127 127 127 231 231 231 F F F 21 420-439 128 128 128 233 233 233 F F F 22 440-459 122 122 122 226 226 226 F F F 23 460-479 122 122 122 226 226 226 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 125 125 125 229 229 229 F F F 26 520-539 128 128 128 234 234 234 F F F 27 540-559 127 127 127 231 231 231 F F F 28 560-579 128 128 128 233 233 233 F F F 29 580-599 128 128 128 233 233 233 F F F Max gen: 128, Max flush: 234 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 233 233 233 F F F 1 020-039 128 128 128 233 233 233 F F F 2 040-059 124 124 124 228 228 228 F F F 3 060-079 128 128 128 232 232 232 F F F 4 080-099 128 128 128 233 233 233 F F F 5 100-119 129 129 129 235 235 235 F F F 6 120-139 126 126 126 230 230 230 F F F 7 140-159 127 127 127 231 231 231 F F F 8 160-179 127 127 127 231 231 231 F F F 9 180-199 122 122 122 226 226 226 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 128 128 128 233 233 233 F F F 12 240-259 127 127 127 231 231 231 F F F 13 260-279 123 123 123 227 227 227 F F F 14 280-299 127 127 127 231 231 231 F F F 15 300-319 123 123 123 227 227 227 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 128 128 128 232 232 232 F F F 18 360-379 122 122 122 225 225 225 F F F 19 380-399 127 127 127 231 231 231 F F F 20 400-419 127 127 127 231 231 231 F F F 21 420-439 128 128 128 233 233 233 F F F 22 440-459 122 122 122 226 226 226 F F F 23 460-479 122 122 122 226 226 226 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 125 125 125 229 229 229 F F F 26 520-539 128 128 128 234 234 234 F F F 27 540-559 127 127 127 231 231 231 F F F 28 560-579 128 128 128 233 233 233 F F F 29 580-599 128 128 128 233 233 233 F F F Max gen: 129, Max flush: 235 Begin loop 65 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.394440004Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:13.437013459Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.43740857Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.468982866Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.469566259Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.469617124Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.469635212Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.469652901Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707} {"msg":"Crucible 07a2f10a-346e-4324-8828-532ed5329fad has session id: ddb38ac4-1727-4024-9647-c9dd8d8439e7","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470166731Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Upstairs opts: Upstairs UUID: 07a2f10a-346e-4324-8828-532ed5329fad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470219405Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Crucible stats registered with UUID: 07a2f10a-346e-4324-8828-532ed5329fad","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470240573Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:13.470256642Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470650393Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470674731Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707} {"msg":"07a2f10a-346e-4324-8828-532ed5329fad active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470732885Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470750834Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"0","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470773111Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"0","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.47078646Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"1","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470800489Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"1","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470813388Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"2","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470827356Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"2","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470840765Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470865152Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.4708926Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:37265)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.470918667Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:58840)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.471157114Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46049)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.47119284Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.47160726Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"1","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.471637847Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"2","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.471653466Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"0","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.472233969Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"1","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.472262876Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"0","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.472279165Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"2","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.472830921Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.472855998Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"[0]R flush_numbers[0..12]: [233, 233, 228, 232, 233, 235, 230, 231, 231, 226, 226, 233]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.472897994Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"[0]R generation[0..12]: [128, 128, 124, 128, 128, 129, 126, 127, 127, 122, 122, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.472917003Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.472931461Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"[1]R flush_numbers[0..12]: [233, 233, 228, 232, 233, 235, 230, 231, 231, 226, 226, 233]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.47294561Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"[1]R generation[0..12]: [128, 128, 124, 128, 128, 129, 126, 127, 127, 122, 122, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.472959188Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.472972727Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"[2]R flush_numbers[0..12]: [233, 233, 228, 232, 233, 235, 230, 231, 231, 226, 226, 233]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.472990256Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"[2]R generation[0..12]: [128, 128, 124, 128, 128, 129, 126, 127, 127, 122, 122, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473004714Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473018413Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473031152Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.47304414Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473056649Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"All extents match"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473081227Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"":"downstairs","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473098535Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473110614Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"0","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473123592Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"1","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473136151Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"client":"2","":"downstairs","session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.47314902Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473161069Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"07a2f10a-346e-4324-8828-532ed5329fad is now active with session: ddb38ac4-1727-4024-9647-c9dd8d8439e7","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473173498Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473185217Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707,"session_id":"ddb38ac4-1727-4024-9647-c9dd8d8439e7"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:13.473197175Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 304 len 32768 data: 10 10 10 12 13 17 17 15 02/30 Write block 196 len 12288 data: 12 12 11 03/30 Read block 412 len 36864 04/30 Write block 528 len 32768 data: 5 9 13 15 14 13 8 10 05/30 Flush 06/30 Write block 151 len 32768 data: 6 6 5 6 6 6 7 12 07/30 Write block 274 len 24576 data: 10 10 11 10 12 11 08/30 Read block 162 len 40960 09/30 Write block 398 len 32768 data: 9 10 12 12 11 15 13 13 10/30 Read block 520 len 4096 11/30 Write block 369 len 16384 data: 6 6 6 5 12/30 Read block 265 len 20480 13/30 Read block 212 len 8192 14/30 Read block 175 len 20480 15/30 Read block 129 len 32768 16/30 Read block 425 len 24576 17/30 Write block 434 len 16384 data: 16 14 13 14 18/30 Read block 201 len 20480 19/30 Read block 95 len 24576 20/30 Read block 309 len 12288 21/30 Write block 372 len 28672 data: 6 4 5 6 7 7 9 22/30 Flush 23/30 Write block 585 len 4096 data: 18 24/30 Read block 77 len 4096 25/30 Read block 386 len 28672 26/30 Write block 74 len 40960 data: 13 12 10 9 10 11 11 10 11 13 27/30 Read block 456 len 40960 28/30 Read block 165 len 8192 29/30 Write block 150 len 20480 data: 9 7 7 6 7 30/30 Write block 367 len 40960 data: 6 7 7 7 7 7 5 6 7 8 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 1 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:13.493638015Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1707} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 233 233 233 F F F 1 020-039 128 128 128 233 233 233 F F F 2 040-059 124 124 124 228 228 228 F F F 3 060-079 128 128 128 232 232 232 T T T 4 080-099 128 128 128 233 233 233 T T T 5 100-119 129 129 129 235 235 235 F F F 6 120-139 126 126 126 230 230 230 F F F 7 140-159 130 130 130 237 237 237 T T T 8 160-179 127 127 127 231 231 231 F F F 9 180-199 130 130 130 236 236 236 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 128 128 128 233 233 233 F F F 12 240-259 127 127 127 231 231 231 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 127 127 127 231 231 231 F F F 15 300-319 130 130 130 236 236 236 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 128 128 128 232 232 232 F F F 18 360-379 130 130 130 237 237 237 T T T 19 380-399 130 130 130 237 237 237 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 122 122 122 226 226 226 F F F 23 460-479 122 122 122 226 226 226 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 125 125 125 229 229 229 F F F 26 520-539 130 130 130 236 236 236 F F F 27 540-559 127 127 127 231 231 231 F F F 28 560-579 128 128 128 233 233 233 F F F 29 580-599 128 128 128 233 233 233 T T T Max gen: 130, Max flush: 237 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 233 233 233 F F F 1 020-039 128 128 128 233 233 233 F F F 2 040-059 124 124 124 228 228 228 F F F 3 060-079 131 131 131 238 238 238 F F F 4 080-099 131 131 131 238 238 238 F F F 5 100-119 129 129 129 235 235 235 F F F 6 120-139 126 126 126 230 230 230 F F F 7 140-159 131 131 131 238 238 238 F F F 8 160-179 127 127 127 231 231 231 F F F 9 180-199 130 130 130 236 236 236 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 128 128 128 233 233 233 F F F 12 240-259 127 127 127 231 231 231 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 127 127 127 231 231 231 F F F 15 300-319 130 130 130 236 236 236 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 128 128 128 232 232 232 F F F 18 360-379 131 131 131 238 238 238 F F F 19 380-399 130 130 130 237 237 237 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 122 122 122 226 226 226 F F F 23 460-479 122 122 122 226 226 226 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 125 125 125 229 229 229 F F F 26 520-539 130 130 130 236 236 236 F F F 27 540-559 127 127 127 231 231 231 F F F 28 560-579 128 128 128 233 233 233 F F F 29 580-599 131 131 131 238 238 238 F F F Max gen: 131, Max flush: 238 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.757583532Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.800668617Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.801267878Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.801307964Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.801322243Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.801336261Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715} {"msg":"Crucible bf8fa159-0ef5-4ef0-8b52-92a46a587214 has session id: 51267024-9e95-427c-a955-f0d9e08448ac","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.80186805Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Upstairs opts: Upstairs UUID: bf8fa159-0ef5-4ef0-8b52-92a46a587214, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.801917795Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Crucible stats registered with UUID: bf8fa159-0ef5-4ef0-8b52-92a46a587214","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.801940622Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:17.8019612Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802327805Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802353302Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715} {"msg":"bf8fa159-0ef5-4ef0-8b52-92a46a587214 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802417686Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802435644Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"0","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802461022Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"0","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.80248548Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"1","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802512767Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"1","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802540614Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"2","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802568352Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"2","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802595729Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802626296Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802651613Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:58130)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802677261Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:62708)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802703028Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:65373)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.802729026Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.803388002Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"1","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.803426158Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"0","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.804102432Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"1","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.804140418Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"0","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.804233849Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"2","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.804711853Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.804734121Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.804806183Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"2","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"[0]R flush_numbers[0..12]: [233, 233, 228, 238, 238, 235, 230, 238, 231, 236, 226, 233]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805332262Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"[0]R generation[0..12]: [128, 128, 124, 131, 131, 129, 126, 131, 127, 130, 122, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.80535556Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805373418Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"[1]R flush_numbers[0..12]: [233, 233, 228, 238, 238, 235, 230, 238, 231, 236, 226, 233]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805387397Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"[1]R generation[0..12]: [128, 128, 124, 131, 131, 129, 126, 131, 127, 130, 122, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805400926Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805414404Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"[2]R flush_numbers[0..12]: [233, 233, 228, 238, 238, 235, 230, 238, 231, 236, 226, 233]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805427703Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"[2]R generation[0..12]: [128, 128, 124, 131, 131, 129, 126, 131, 127, 130, 122, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.80545272Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:14:17.805467519Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805485737Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805498666Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805511235Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805523694Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"":"downstairs","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805536892Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805548831Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"0","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.80556202Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"1","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805576908Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"client":"2","":"downstairs","session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805590067Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805602016Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"bf8fa159-0ef5-4ef0-8b52-92a46a587214 is now active with session: 51267024-9e95-427c-a955-f0d9e08448ac","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805614315Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805625914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715,"session_id":"51267024-9e95-427c-a955-f0d9e08448ac"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:17.805637773Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 572 len 28672 data: 13 13 11 9 7 13 15 02/30 Read block 413 len 20480 03/30 Read block 199 len 36864 04/30 Write block 590 len 20480 data: 12 12 12 12 10 05/30 Read block 381 len 16384 06/30 Flush 07/30 Write block 387 len 20480 data: 11 11 10 9 10 08/30 Read block 448 len 8192 09/30 Write block 231 len 40960 data: 8 6 7 10 11 12 14 16 16 16 10/30 Read block 525 len 40960 11/30 Read block 531 len 16384 12/30 Read block 523 len 28672 13/30 Read block 169 len 20480 14/30 Read block 296 len 8192 15/30 Flush 16/30 Write block 62 len 12288 data: 10 10 9 17/30 Write block 463 len 12288 data: 13 13 14 18/30 Read block 545 len 36864 19/30 Write block 582 len 4096 data: 17 20/30 Read block 482 len 12288 21/30 Flush 22/30 Write block 86 len 24576 data: 10 11 11 13 13 12 23/30 Read block 448 len 36864 24/30 Read block 11 len 16384 25/30 Flush 26/30 Write block 159 len 28672 data: 11 11 12 13 15 18 14 27/30 Write block 344 len 16384 data: 12 12 14 14 28/30 Read block 488 len 12288 29/30 Read block 169 len 16384 30/30 Write block 252 len 16384 data: 13 13 13 10 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:17.825286299Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1715} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 233 233 233 F F F 1 020-039 128 128 128 233 233 233 F F F 2 040-059 124 124 124 228 228 228 F F F 3 060-079 132 132 132 241 241 241 F F F 4 080-099 132 132 132 242 242 242 F F F 5 100-119 129 129 129 235 235 235 F F F 6 120-139 126 126 126 230 230 230 F F F 7 140-159 131 131 131 238 238 238 T T T 8 160-179 127 127 127 231 231 231 T T T 9 180-199 130 130 130 236 236 236 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 132 132 132 240 240 240 F F F 12 240-259 132 132 132 240 240 240 T T T 13 260-279 130 130 130 237 237 237 F F F 14 280-299 127 127 127 231 231 231 F F F 15 300-319 130 130 130 236 236 236 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 128 128 128 232 232 232 T T T 18 360-379 131 131 131 238 238 238 F F F 19 380-399 132 132 132 240 240 240 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 122 122 122 226 226 226 F F F 23 460-479 132 132 132 241 241 241 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 125 125 125 229 229 229 F F F 26 520-539 130 130 130 236 236 236 F F F 27 540-559 127 127 127 231 231 231 F F F 28 560-579 132 132 132 239 239 239 F F F 29 580-599 132 132 132 241 241 241 F F F Max gen: 132, Max flush: 242 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 233 233 233 F F F 1 020-039 128 128 128 233 233 233 F F F 2 040-059 124 124 124 228 228 228 F F F 3 060-079 132 132 132 241 241 241 F F F 4 080-099 132 132 132 242 242 242 F F F 5 100-119 129 129 129 235 235 235 F F F 6 120-139 126 126 126 230 230 230 F F F 7 140-159 133 133 133 243 243 243 F F F 8 160-179 133 133 133 243 243 243 F F F 9 180-199 130 130 130 236 236 236 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 132 132 132 240 240 240 F F F 12 240-259 133 133 133 243 243 243 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 127 127 127 231 231 231 F F F 15 300-319 130 130 130 236 236 236 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 133 133 133 243 243 243 F F F 18 360-379 131 131 131 238 238 238 F F F 19 380-399 132 132 132 240 240 240 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 122 122 122 226 226 226 F F F 23 460-479 132 132 132 241 241 241 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 125 125 125 229 229 229 F F F 26 520-539 130 130 130 236 236 236 F F F 27 540-559 127 127 127 231 231 231 F F F 28 560-579 132 132 132 239 239 239 F F F 29 580-599 132 132 132 241 241 241 F F F Max gen: 133, Max flush: 243 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.053236442Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:22.095847863Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.096261643Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.127851307Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.128457178Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.128501883Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.128519442Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.12853743Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723} {"msg":"Crucible 0cc981fe-08c9-4868-b6fc-3d0c543b9a77 has session id: 13a89fc5-7494-4c54-9f9a-2e3e2ff321ad","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129182437Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Upstairs opts: Upstairs UUID: 0cc981fe-08c9-4868-b6fc-3d0c543b9a77, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129216954Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Crucible stats registered with UUID: 0cc981fe-08c9-4868-b6fc-3d0c543b9a77","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129231932Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:22.129244511Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129607556Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129631113Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723} {"msg":"0cc981fe-08c9-4868-b6fc-3d0c543b9a77 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129691137Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129709296Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"0","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129735193Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"0","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129749372Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"1","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.12976648Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"1","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129792018Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"2","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129818345Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"2","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129844773Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.12987075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129897477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:45875)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129925545Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:49164)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.129951642Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:55364)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.12997934Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.130632086Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"2","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.130661933Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"1","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.130685101Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"0","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.13120885Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"1","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.131246696Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"0","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.131276713Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"2","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.131778944Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.131804152Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"[0]R flush_numbers[0..12]: [233, 233, 228, 241, 242, 235, 230, 243, 243, 236, 226, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.131831869Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"[0]R generation[0..12]: [128, 128, 124, 132, 132, 129, 126, 133, 133, 130, 122, 132]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.131846568Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.131860876Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"[1]R flush_numbers[0..12]: [233, 233, 228, 241, 242, 235, 230, 243, 243, 236, 226, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.131874715Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"[1]R generation[0..12]: [128, 128, 124, 132, 132, 129, 126, 133, 133, 130, 122, 132]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.131888314Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.131904742Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"[2]R flush_numbers[0..12]: [233, 233, 228, 241, 242, 235, 230, 243, 243, 236, 226, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.13192961Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"[2]R generation[0..12]: [128, 128, 124, 132, 132, 129, 126, 133, 133, 130, 122, 132]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.131957177Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":Wait for a query_work_queue command to finish before sending IO 1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.131990074Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.132019751Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.132043848Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.132067476Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.132091594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"":"downstairs","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.132116791Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.1321308Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"0","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.132144129Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"1","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-28T17:14:22.132164797Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"client":"2","":"downstairs","session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.132205013Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.132226131Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"0cc981fe-08c9-4868-b6fc-3d0c543b9a77 is now active with session: 13a89fc5-7494-4c54-9f9a-2e3e2ff321ad","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.132248099Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.132268866Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723,"session_id":"13a89fc5-7494-4c54-9f9a-2e3e2ff321ad"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:22.132289395Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 378 len 24576 data: 10 9 8 9 11 12 02/30 Write block 576 len 40960 data: 8 14 16 12 13 14 18 18 19 19 03/30 Write block 356 len 36864 data: 10 12 14 13 10 9 6 6 7 04/30 Read block 145 len 28672 05/30 Write block 24 len 28672 data: 9 10 11 10 10 10 9 06/30 Read block 95 len 4096 07/30 Read block 17 len 36864 08/30 Flush 09/30 Write block 419 len 12288 data: 8 8 9 10/30 Read block 368 len 12288 11/30 Write block 47 len 32768 data: 8 6 6 6 9 11 9 9 12/30 Read block 305 len 12288 13/30 Flush 14/30 Write block 5 len 36864 data: 7 6 9 10 11 11 12 12 13 15/30 Write block 531 len 28672 data: 16 15 14 9 11 8 10 16/30 Write block 468 len 32768 data: 7 8 9 9 8 8 5 5 17/30 Read block 108 len 40960 18/30 Read block 71 len 36864 19/30 Read block 375 len 8192 20/30 Write block 344 len 24576 data: 13 13 15 15 14 13 21/30 Read block 236 len 24576 22/30 Read block 451 len 36864 23/30 Write block 587 len 20480 data: 10 11 14 13 13 24/30 Read block 409 len 12288 25/30 Write block 220 len 28672 data: 8 8 12 13 14 14 13 26/30 Read block 417 len 24576 27/30 Write block 440 len 16384 data: 13 11 11 13 28/30 Write block 298 len 36864 data: 10 9 13 14 11 12 11 11 11 29/30 Write block 228 len 4096 data: 12 30/30 Read block 419 len 32768 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 17 17 16 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:22.153059162Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1723} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 233 233 233 T T T 1 020-039 134 134 134 244 244 244 F F F 2 040-059 134 134 134 245 245 245 F F F 3 060-079 132 132 132 241 241 241 F F F 4 080-099 132 132 132 242 242 242 F F F 5 100-119 129 129 129 235 235 235 F F F 6 120-139 126 126 126 230 230 230 F F F 7 140-159 133 133 133 243 243 243 F F F 8 160-179 133 133 133 243 243 243 F F F 9 180-199 130 130 130 236 236 236 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 132 132 132 240 240 240 T T T 12 240-259 133 133 133 243 243 243 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 127 127 127 231 231 231 T T T 15 300-319 130 130 130 236 236 236 T T T 16 320-339 124 124 124 228 228 228 F F F 17 340-359 134 134 134 244 244 244 T T T 18 360-379 134 134 134 244 244 244 F F F 19 380-399 134 134 134 244 244 244 F F F 20 400-419 134 134 134 245 245 245 F F F 21 420-439 134 134 134 245 245 245 F F F 22 440-459 122 122 122 226 226 226 T T T 23 460-479 132 132 132 241 241 241 T T T 24 480-499 127 127 127 231 231 231 F F F 25 500-519 125 125 125 229 229 229 F F F 26 520-539 130 130 130 236 236 236 T T T 27 540-559 127 127 127 231 231 231 F F F 28 560-579 134 134 134 244 244 244 F F F 29 580-599 134 134 134 244 244 244 T T T Max gen: 134, Max flush: 245 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 246 246 246 F F F 1 020-039 134 134 134 244 244 244 F F F 2 040-059 134 134 134 245 245 245 F F F 3 060-079 132 132 132 241 241 241 F F F 4 080-099 132 132 132 242 242 242 F F F 5 100-119 129 129 129 235 235 235 F F F 6 120-139 126 126 126 230 230 230 F F F 7 140-159 133 133 133 243 243 243 F F F 8 160-179 133 133 133 243 243 243 F F F 9 180-199 130 130 130 236 236 236 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 135 135 135 246 246 246 F F F 12 240-259 133 133 133 243 243 243 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 135 135 135 246 246 246 F F F 15 300-319 135 135 135 246 246 246 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 135 135 135 246 246 246 F F F 18 360-379 134 134 134 244 244 244 F F F 19 380-399 134 134 134 244 244 244 F F F 20 400-419 134 134 134 245 245 245 F F F 21 420-439 134 134 134 245 245 245 F F F 22 440-459 135 135 135 246 246 246 F F F 23 460-479 135 135 135 246 246 246 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 125 125 125 229 229 229 F F F 26 520-539 135 135 135 246 246 246 F F F 27 540-559 127 127 127 231 231 231 F F F 28 560-579 134 134 134 244 244 244 F F F 29 580-599 135 135 135 246 246 246 F F F Max gen: 135, Max flush: 246 Begin loop 68 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.4401063Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.484193498Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.484770191Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.484811018Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.484825396Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.484839515Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731} {"msg":"Crucible aa1ce179-cdf6-49de-aad9-1b7833f936e1 has session id: 23cab440-1608-410c-ab16-13995dbdc153","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.485415569Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Upstairs opts: Upstairs UUID: aa1ce179-cdf6-49de-aad9-1b7833f936e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.485459664Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Crucible stats registered with UUID: aa1ce179-cdf6-49de-aad9-1b7833f936e1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.485482082Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:26.48550263Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.485867775Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.485895022Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731} {"msg":"aa1ce179-cdf6-49de-aad9-1b7833f936e1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.485950547Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.485968745Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"0","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.485994372Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"0","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.48601893Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"1","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.486045877Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"1","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.486072985Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"2","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.486100982Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"2","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.486126979Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.486154837Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.486185354Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"ds_connection connected from Ok(127.0.0.1:64770)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.486293703Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"io task","client":"0","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"ds_connection connected from Ok(127.0.0.1:34358)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.48632044Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"io task","client":"2","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"ds_connection connected from Ok(127.0.0.1:39587)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.486335129Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"io task","client":"1","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.486934251Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"0","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.486975177Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"2","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.487017113Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"1","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.487515784Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"0","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.4875568Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"2","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.487586917Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"1","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488053022Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488079069Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"[0]R flush_numbers[0..12]: [246, 244, 245, 241, 242, 235, 230, 243, 243, 236, 226, 246]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488096108Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"[0]R generation[0..12]: [135, 134, 134, 132, 132, 129, 126, 133, 133, 130, 122, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488110746Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488125055Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"[1]R flush_numbers[0..12]: [246, 244, 245, 241, 242, 235, 230, 243, 243, 236, 226, 246]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488142353Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"[1]R generation[0..12]: [135, 134, 134, 132, 132, 129, 126, 133, 133, 130, 122, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488156122Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488169831Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"[2]R flush_numbers[0..12]: [246, 244, 245, 241, 242, 235, 230, 243, 243, 236, 226, 246]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488183409Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"[2]R generation[0..12]: [135, 134, 134, 132, 132, 129, 126, 133, 133, 130, 122, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488197288Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488212036Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488224865Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488237674Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488250073Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488269421Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"":"downstairs","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.48828244Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488294268Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"0","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488307207Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"1","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488319916Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"client":"2","":"downstairs","session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488334954Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488347053Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"aa1ce179-cdf6-49de-aad9-1b7833f936e1 is now active with session: 23cab440-1608-410c-ab16-13995dbdc153","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488359502Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.488371161Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731,"session_id":"23cab440-1608-410c-ab16-13995dbdc153"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:26.48838322Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 502 len 8192 data: 9 12 02/30 Write block 585 len 4096 data: 20 03/30 Read block 423 len 36864 04/30 Write block 587 len 28672 data: 11 12 15 14 14 13 13 05/30 Read block 288 len 36864 06/30 Write block 533 len 36864 data: 15 10 12 9 11 11 12 13 11 07/30 Write block 31 len 8192 data: 8 8 08/30 Write block 109 len 28672 data: 9 8 6 8 11 12 11 09/30 Write block 71 len 12288 data: 14 15 14 10/30 Read block 101 len 28672 11/30 Read block 311 len 12288 12/30 Read block 73 len 20480 13/30 Write block 134 len 20480 data: 9 9 9 12 13 14/30 Read block 385 len 4096 15/30 Write block 558 len 12288 data: 14 11 11 16/30 Write block 397 len 36864 data: 8 10 11 13 13 12 16 14 14 17/30 Write block 43 len 36864 data: 9 9 7 7 9 7 7 7 10 18/30 Read block 322 len 40960 19/30 Write block 105 len 36864 data: 9 9 12 11 10 9 7 9 12 20/30 Write block 170 len 32768 data: 11 10 14 14 14 15 14 14 21/30 Read block 146 len 16384 22/30 Read block 397 len 40960 23/30 Write block 166 len 24576 data: 12 12 13 12 12 11 24/30 Read block 410 len 28672 25/30 Write block 126 len 28672 data: 5 4 3 3 3 9 10 26/30 Read block 67 len 4096 27/30 Read block 521 len 24576 28/30 Write block 549 len 16384 data: 11 12 12 13 29/30 Write block 235 len 4096 data: 12 30/30 Write block 424 len 36864 data: 8 8 10 10 11 11 12 12 11 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:26.507131524Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1731} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 246 246 246 F F F 1 020-039 134 134 134 244 244 244 T T T 2 040-059 134 134 134 245 245 245 T T T 3 060-079 132 132 132 241 241 241 T T T 4 080-099 132 132 132 242 242 242 F F F 5 100-119 129 129 129 235 235 235 T T T 6 120-139 126 126 126 230 230 230 T T T 7 140-159 133 133 133 243 243 243 F F F 8 160-179 133 133 133 243 243 243 T T T 9 180-199 130 130 130 236 236 236 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 135 135 135 246 246 246 T T T 12 240-259 133 133 133 243 243 243 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 135 135 135 246 246 246 F F F 15 300-319 135 135 135 246 246 246 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 135 135 135 246 246 246 F F F 18 360-379 134 134 134 244 244 244 F F F 19 380-399 134 134 134 244 244 244 T T T 20 400-419 134 134 134 245 245 245 T T T 21 420-439 134 134 134 245 245 245 T T T 22 440-459 135 135 135 246 246 246 F F F 23 460-479 135 135 135 246 246 246 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 125 125 125 229 229 229 T T T 26 520-539 135 135 135 246 246 246 T T T 27 540-559 127 127 127 231 231 231 T T T 28 560-579 134 134 134 244 244 244 T T T 29 580-599 135 135 135 246 246 246 T T T Max gen: 135, Max flush: 246 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 246 246 246 F F F 1 020-039 136 136 136 247 247 247 F F F 2 040-059 136 136 136 247 247 247 F F F 3 060-079 136 136 136 247 247 247 F F F 4 080-099 132 132 132 242 242 242 F F F 5 100-119 136 136 136 247 247 247 F F F 6 120-139 136 136 136 247 247 247 F F F 7 140-159 133 133 133 243 243 243 F F F 8 160-179 136 136 136 247 247 247 F F F 9 180-199 130 130 130 236 236 236 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 136 136 136 247 247 247 F F F 12 240-259 133 133 133 243 243 243 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 135 135 135 246 246 246 F F F 15 300-319 135 135 135 246 246 246 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 135 135 135 246 246 246 F F F 18 360-379 134 134 134 244 244 244 F F F 19 380-399 136 136 136 247 247 247 F F F 20 400-419 136 136 136 247 247 247 F F F 21 420-439 136 136 136 247 247 247 F F F 22 440-459 135 135 135 246 246 246 F F F 23 460-479 135 135 135 246 246 246 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 136 136 136 247 247 247 F F F 26 520-539 136 136 136 247 247 247 F F F 27 540-559 136 136 136 247 247 247 F F F 28 560-579 136 136 136 247 247 247 F F F 29 580-599 136 136 136 247 247 247 F F F Max gen: 136, Max flush: 247 Begin loop 69 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.792052901Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:40.833651231Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.834029584Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.86467326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.865232685Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.865266812Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.865280901Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.865294699Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739} {"msg":"Crucible c24c0fd1-3e9a-4e9b-916d-7dc1f7584839 has session id: 110389c7-60b7-4d8f-a545-1730ccb5643b","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.865931528Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Upstairs opts: Upstairs UUID: c24c0fd1-3e9a-4e9b-916d-7dc1f7584839, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.865977403Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Crucible stats registered with UUID: c24c0fd1-3e9a-4e9b-916d-7dc1f7584839","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.865999601Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:40.86601282Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866425249Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866449957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739} {"msg":"c24c0fd1-3e9a-4e9b-916d-7dc1f7584839 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866512021Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866530159Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"0","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866551837Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"0","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866565786Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"1","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866579714Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"1","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866592483Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"2","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866606142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"2","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866618661Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866632029Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866646868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"ds_connection connected from Ok(127.0.0.1:54901)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866670416Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"io task","client":"2","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42683)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.866884995Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"io task","client":"1","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64277)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.86693159Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"io task","client":"0","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.8674505Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"2","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.867478437Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"1","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.867494125Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"0","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868104826Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"1","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868145742Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"2","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868177459Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"0","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868747383Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868772181Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"[0]R flush_numbers[0..12]: [246, 247, 247, 247, 242, 247, 247, 243, 247, 236, 226, 247]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868791829Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"[0]R generation[0..12]: [135, 136, 136, 136, 132, 136, 136, 133, 136, 130, 122, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868806288Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868820976Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"[1]R flush_numbers[0..12]: [246, 247, 247, 247, 242, 247, 247, 243, 247, 236, 226, 247]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868835165Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"[1]R generation[0..12]: [135, 136, 136, 136, 132, 136, 136, 133, 136, 130, 122, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868848544Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868866052Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"[2]R flush_numbers[0..12]: [246, 247, 247, 247, 242, 247, 247, 243, 247, 236, 226, 247]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.86887999Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"[2]R generation[0..12]: [135, 136, 136, 136, 132, 136, 136, 133, 136, 130, 122, 136]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868893559Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868907218Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 137","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868925726Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Generation requested: 138 >= found:137","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868939635Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868962592Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868983Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"":"downstairs","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.868996839Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.869008808Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"0","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.869021907Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"1","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.869039735Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"client":"2","":"downstairs","session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.869062963Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.869076751Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"c24c0fd1-3e9a-4e9b-916d-7dc1f7584839 is now active with session: 110389c7-60b7-4d8f-a545-1730ccb5643b","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.86908935Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.869111228Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739,"session_id":"110389c7-60b7-4d8f-a545-1730ccb5643b"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:40.869135796Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 19 len 28672 data: 10 9 7 9 10 10 11 02/30 Read block 221 len 4096 03/30 Write block 541 len 36864 data: 12 8 7 8 11 11 8 10 12 04/30 Read block 284 len 4096 05/30 Write block 540 len 32768 data: 14 13 9 8 9 12 12 9 06/30 Read block 316 len 4096 07/30 Read block 95 len 28672 08/30 Write block 543 len 4096 data: 9 09/30 Read block 508 len 32768 10/30 Flush 11/30 Write block 31 len 4096 data: 9 12/30 Write block 412 len 16384 data: 8 8 8 8 13/30 Read block 560 len 40960 14/30 Read block 567 len 40960 15/30 Write block 236 len 24576 data: 13 15 17 17 17 15 16/30 Flush 17/30 Write block 576 len 16384 data: 9 15 17 13 18/30 Read block 229 len 20480 19/30 Read block 1 len 4096 20/30 Write block 55 len 16384 data: 8 5 6 6 21/30 Read block 138 len 32768 22/30 Write block 503 len 36864 data: 13 13 14 13 10 10 13 14 14 23/30 Read block 239 len 40960 24/30 Read block 307 len 32768 25/30 Read block 293 len 12288 26/30 Write block 353 len 8192 data: 13 12 27/30 Write block 442 len 8192 data: 12 14 28/30 Read block 57 len 16384 29/30 Write block 450 len 12288 data: 13 10 7 30/30 Read block 246 len 32768 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 4 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 12 14 14 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:40.889025799Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1739} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 248 248 248 F F F 1 020-039 138 138 138 249 249 249 F F F 2 040-059 136 136 136 247 247 247 T T T 3 060-079 136 136 136 247 247 247 F F F 4 080-099 132 132 132 242 242 242 F F F 5 100-119 136 136 136 247 247 247 F F F 6 120-139 136 136 136 247 247 247 F F F 7 140-159 133 133 133 243 243 243 F F F 8 160-179 136 136 136 247 247 247 F F F 9 180-199 130 130 130 236 236 236 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 138 138 138 249 249 249 F F F 12 240-259 138 138 138 249 249 249 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 135 135 135 246 246 246 F F F 15 300-319 135 135 135 246 246 246 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 135 135 135 246 246 246 T T T 18 360-379 134 134 134 244 244 244 F F F 19 380-399 136 136 136 247 247 247 F F F 20 400-419 138 138 138 249 249 249 F F F 21 420-439 136 136 136 247 247 247 F F F 22 440-459 135 135 135 246 246 246 T T T 23 460-479 135 135 135 246 246 246 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 136 136 136 247 247 247 T T T 26 520-539 136 136 136 247 247 247 F F F 27 540-559 138 138 138 248 248 248 F F F 28 560-579 136 136 136 247 247 247 T T T 29 580-599 136 136 136 247 247 247 F F F Max gen: 138, Max flush: 249 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 248 248 248 F F F 1 020-039 138 138 138 249 249 249 F F F 2 040-059 139 139 139 250 250 250 F F F 3 060-079 136 136 136 247 247 247 F F F 4 080-099 132 132 132 242 242 242 F F F 5 100-119 136 136 136 247 247 247 F F F 6 120-139 136 136 136 247 247 247 F F F 7 140-159 133 133 133 243 243 243 F F F 8 160-179 136 136 136 247 247 247 F F F 9 180-199 130 130 130 236 236 236 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 138 138 138 249 249 249 F F F 12 240-259 138 138 138 249 249 249 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 135 135 135 246 246 246 F F F 15 300-319 135 135 135 246 246 246 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 139 139 139 250 250 250 F F F 18 360-379 134 134 134 244 244 244 F F F 19 380-399 136 136 136 247 247 247 F F F 20 400-419 138 138 138 249 249 249 F F F 21 420-439 136 136 136 247 247 247 F F F 22 440-459 139 139 139 250 250 250 F F F 23 460-479 135 135 135 246 246 246 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 139 139 139 250 250 250 F F F 26 520-539 136 136 136 247 247 247 F F F 27 540-559 138 138 138 248 248 248 F F F 28 560-579 139 139 139 250 250 250 F F F 29 580-599 136 136 136 247 247 247 F F F Max gen: 139, Max flush: 250 Begin loop 70 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.158319386Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.202261487Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.202894315Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.202940141Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.202960059Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.202975008Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747} {"msg":"Crucible bc1d2b69-ebf0-4dd1-a59b-75c898203742 has session id: 72938d63-fa5b-48ad-a418-0a5689e6ccec","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.203521625Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Upstairs opts: Upstairs UUID: bc1d2b69-ebf0-4dd1-a59b-75c898203742, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.203559481Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Crucible stats registered with UUID: bc1d2b69-ebf0-4dd1-a59b-75c898203742","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.203575529Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:45.203588248Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.20398183Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.204010837Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747} {"msg":"bc1d2b69-ebf0-4dd1-a59b-75c898203742 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.204067741Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.20408587Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"0","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.204107037Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"0","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.204120736Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"1","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.204134685Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"1","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.204147793Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"2","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.204165832Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"2","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.20418722Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.204214237Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.204240775Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"ds_connection connected from Ok(127.0.0.1:40675)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.204276071Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"io task","client":"2","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"ds_connection connected from Ok(127.0.0.1:57467)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.204461363Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"io task","client":"1","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"ds_connection connected from Ok(127.0.0.1:50690)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.204500919Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"io task","client":"0","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.205010289Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"1","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.205052785Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"2","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.205070254Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"0","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.205573495Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"1","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.205679804Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"0","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.205768976Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"2","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206222422Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.20634206Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"[0]R flush_numbers[0..12]: [248, 249, 250, 247, 242, 247, 247, 243, 247, 236, 226, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206374567Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"[0]R generation[0..12]: [138, 138, 139, 136, 132, 136, 136, 133, 136, 130, 122, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206390785Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206405494Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"[1]R flush_numbers[0..12]: [248, 249, 250, 247, 242, 247, 247, 243, 247, 236, 226, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206419142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"[1]R generation[0..12]: [138, 138, 139, 136, 132, 136, 136, 133, 136, 130, 122, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206435081Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206460018Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"[2]R flush_numbers[0..12]: [248, 249, 250, 247, 242, 247, 247, 243, 247, 236, 226, 249]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206487476Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"[2]R generation[0..12]: [138, 138, 139, 136, 132, 136, 136, 133, 136, 130, 122, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206513783Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28",Wait for a query_work_queue command to finish before sending IO "pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.20654562Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206563858Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206577767Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206590596Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206603384Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"":"downstairs","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206619853Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206632091Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"0","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.20664525Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"1","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206658079Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"client":"2","":"downstairs","session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206671088Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206683047Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"bc1d2b69-ebf0-4dd1-a59b-75c898203742 is now active with session: 72938d63-fa5b-48ad-a418-0a5689e6ccec","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206695545Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206707204Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747,"session_id":"72938d63-fa5b-48ad-a418-0a5689e6ccec"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:45.206719073Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 181 len 32768 data: 13 13 14 12 11 9 10 8 02/30 Read block 311 len 4096 03/30 Write block 367 len 24576 data: 7 8 8 8 8 8 04/30 Flush 05/30 Write block 588 len 20480 data: 13 16 15 15 14 06/30 Read block 112 len 28672 07/30 Read block 131 len 36864 08/30 Flush 09/30 Write block 195 len 4096 data: 12 10/30 Flush 11/30 Write block 420 len 4096 data: 9 12/30 Write block 430 len 24576 data: 13 13 12 14 17 15 13/30 Read block 584 len 16384 14/30 Write block 164 len 8192 data: 19 15 15/30 Write block 153 len 24576 data: 7 8 7 7 8 13 16/30 Write block 230 len 4096 data: 10 17/30 Read block 546 len 4096 18/30 Write block 108 len 16384 data: 12 11 10 8 19/30 Write block 585 len 4096 data: 21 20/30 Write block 140 len 8192 data: 18 15 21/30 Read block 266 len 24576 22/30 Read block 112 len 12288 23/30 Write block 290 len 28672 data: 11 10 10 9 10 8 10 24/30 Read block 296 len 24576 25/30 Read block 265 len 32768 26/30 Flush 27/30 Write block 308 len 8192 data: 14 18 28/30 Read block 98 len 16384 29/30 Read block 528 len 12288 30/30 Write block 542 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 3 3 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:45.224157325Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1747} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 248 248 248 F F F 1 020-039 138 138 138 249 249 249 F F F 2 040-059 139 139 139 250 250 250 F F F 3 060-079 136 136 136 247 247 247 F F F 4 080-099 132 132 132 242 242 242 F F F 5 100-119 140 140 140 254 254 254 F F F 6 120-139 136 136 136 247 247 247 F F F 7 140-159 140 140 140 254 254 254 F F F 8 160-179 140 140 140 254 254 254 F F F 9 180-199 140 140 140 253 253 253 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 140 140 140 254 254 254 F F F 12 240-259 138 138 138 249 249 249 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 140 140 140 254 254 254 F F F 15 300-319 135 135 135 246 246 246 T T T 16 320-339 124 124 124 228 228 228 F F F 17 340-359 139 139 139 250 250 250 F F F 18 360-379 140 140 140 251 251 251 F F F 19 380-399 136 136 136 247 247 247 F F F 20 400-419 138 138 138 249 249 249 F F F 21 420-439 140 140 140 254 254 254 F F F 22 440-459 139 139 139 250 250 250 F F F 23 460-479 135 135 135 246 246 246 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 139 139 139 250 250 250 F F F 26 520-539 136 136 136 247 247 247 F F F 27 540-559 138 138 138 248 248 248 T T T 28 560-579 139 139 139 250 250 250 F F F 29 580-599 140 140 140 254 254 254 F F F Max gen: 140, Max flush: 254 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 248 248 248 F F F 1 020-039 138 138 138 249 249 249 F F F 2 040-059 139 139 139 250 250 250 F F F 3 060-079 136 136 136 247 247 247 F F F 4 080-099 132 132 132 242 242 242 F F F 5 100-119 140 140 140 254 254 254 F F F 6 120-139 136 136 136 247 247 247 F F F 7 140-159 140 140 140 254 254 254 F F F 8 160-179 140 140 140 254 254 254 F F F 9 180-199 140 140 140 253 253 253 F F F 10 200-219 122 122 122 226 226 226 F F F 11 220-239 140 140 140 254 254 254 F F F 12 240-259 138 138 138 249 249 249 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 140 140 140 254 254 254 F F F 15 300-319 141 141 141 255 255 255 F F F 16 320-339 124 124 124 228 228 228 F F F 17 340-359 139 139 139 250 250 250 F F F 18 360-379 140 140 140 251 251 251 F F F 19 380-399 136 136 136 247 247 247 F F F 20 400-419 138 138 138 249 249 249 F F F 21 420-439 140 140 140 254 254 254 F F F 22 440-459 139 139 139 250 250 250 F F F 23 460-479 135 135 135 246 246 246 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 139 139 139 250 250 250 F F F 26 520-539 136 136 136 247 247 247 F F F 27 540-559 141 141 141 255 255 255 F F F 28 560-579 139 139 139 250 250 250 F F F 29 580-599 140 140 140 254 254 254 F F F Max gen: 141, Max flush: 255 Begin loop 71 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.439876858Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:49.481578417Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.481954791Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.512791188Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.513348434Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.51338279Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.513401029Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.513415467Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755} {"msg":"Crucible d4351392-3745-45c7-ae54-ca07237587d1 has session id: 0e04b1bd-2962-47cd-b7cf-3e2807fca1de","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.513944706Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Upstairs opts: Upstairs UUID: d4351392-3745-45c7-ae54-ca07237587d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.513980782Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Crucible stats registered with UUID: d4351392-3745-45c7-ae54-ca07237587d1","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.513995691Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:49.514008509Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514441567Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514470444Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755} {"msg":"d4351392-3745-45c7-ae54-ca07237587d1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.51452345Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514541538Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"0","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514564046Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"0","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514578264Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"1","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514592573Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"1","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514605762Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"2","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514627289Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"2","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514653387Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514672505Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514686223Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"ds_connection connected from Ok(127.0.0.1:47298)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514699762Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"io task","client":"0","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"ds_connection connected from Ok(127.0.0.1:34514)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514842078Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"io task","client":"1","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"ds_connection connected from Ok(127.0.0.1:46517)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.514943649Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"io task","client":"2","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.51544117Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"1","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.515620823Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"2","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.515662788Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"0","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.516107955Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"1","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.516137122Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"2","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.516396587Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"0","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.516708787Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.516745123Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"[0]R flush_numbers[0..12]: [248, 249, 250, 247, 242, 254, 247, 254, 254, 253, 226, 254]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.51687744Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"[0]R generation[0..12]: [138, 138, 139, 136, 132, 140, 136, 140, 140, 140, 122, 140]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.516897928Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.516912787Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"[1]R flush_numbers[0..12]: [248, 249, 250, 247, 242, 254, 247, 254, 254, 253, 226, 254]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.516926885Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"[1]R generation[0..12]: [138, 138, 139, 136, 132, 140, 136, 140, 140, 140, 122, 140]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.516940664Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.516954673Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"[2]R flush_numbers[0..12]: [248, 249, 250, 247, 242, 254, 247, 254, 254, 253, 226, 254]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.516968111Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"[2]R generation[0..12]: [138, 138, 139, 136, 132, 140, 136, 140, 140, 140, 122, 140]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.51698573Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO ":30,"time":"2025-01-28T17:14:49.517021326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.517046504Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.517072031Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Next flush: 256","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.517096059Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.517120697Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"":"downstairs","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.517147334Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.517171142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"0","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.517198279Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"1","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.517225046Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"client":"2","":"downstairs","session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"level":30,"time":"2025-01-28T17:14:49.517254333Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.517284861Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"d4351392-3745-45c7-ae54-ca07237587d1 is now active with session: 0e04b1bd-2962-47cd-b7cf-3e2807fca1de","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.517309238Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.517325167Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755,"session_id":"0e04b1bd-2962-47cd-b7cf-3e2807fca1de"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:49.517338175Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 201 len 20480 data: 10 11 13 14 13 02/30 Read block 392 len 40960 03/30 Read block 222 len 36864 04/30 Write block 79 len 12288 data: 12 12 11 05/30 Write block 397 len 40960 data: 9 11 12 14 14 13 17 15 15 13 06/30 Write block 527 len 20480 data: 7 6 10 14 17 07/30 Read block 287 len 4096 08/30 Flush 09/30 Write block 330 len 8192 data: 13 9 10/30 Write block 570 len 20480 data: 14 14 14 14 12 11/30 Read block 403 len 36864 12/30 Write block 153 len 12288 data: 8 9 8 13/30 Flush 14/30 Write block 399 len 4096 data: 13 15/30 Read block 581 len 32768 16/30 Read block 283 len 32768 17/30 Write block 179 len 8192 data: 16 17 18/30 Read block 579 len 36864 19/30 Read block 496 len 36864 20/30 Read block 580 len 12288 21/30 Write block 182 len 40960 data: 14 15 13 12 10 11 9 9 10 12 22/30 Flush 23/30 Write block 589 len 24576 data: 17 16 16 15 14 11 24/30 Read block 469 len 36864 25/30 Flush 26/30 Write block 129 len 24576 data: 4 4 10 11 11 10 27/30 Write block 441 len 28672 data: 12 13 15 15 17 16 16 28/30 Write block 293 len 24576 data: 10 11 9 11 10 11 29/30 Write block 137 len 24576 data: 13 14 15 19 16 13 30/30 Write block 66 len 8192 data: 12 13 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:49.536048494Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1755} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 248 248 248 F F F 1 020-039 138 138 138 249 249 249 F F F 2 040-059 139 139 139 250 250 250 F F F 3 060-079 142 142 142 256 256 256 T T T 4 080-099 142 142 142 256 256 256 F F F 5 100-119 140 140 140 254 254 254 F F F 6 120-139 136 136 136 247 247 247 T T T 7 140-159 142 142 142 257 257 257 T T T 8 160-179 142 142 142 258 258 258 F F F 9 180-199 142 142 142 258 258 258 F F F 10 200-219 142 142 142 256 256 256 F F F 11 220-239 140 140 140 254 254 254 F F F 12 240-259 138 138 138 249 249 249 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 140 140 140 254 254 254 T T T 15 300-319 141 141 141 255 255 255 F F F 16 320-339 142 142 142 257 257 257 F F F 17 340-359 139 139 139 250 250 250 F F F 18 360-379 140 140 140 251 251 251 F F F 19 380-399 142 142 142 258 258 258 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 140 140 140 254 254 254 F F F 22 440-459 139 139 139 250 250 250 T T T 23 460-479 135 135 135 246 246 246 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 139 139 139 250 250 250 F F F 26 520-539 142 142 142 256 256 256 F F F 27 540-559 141 141 141 255 255 255 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 142 142 142 259 259 259 F F F Max gen: 142, Max flush: 259 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 248 248 248 F F F 1 020-039 138 138 138 249 249 249 F F F 2 040-059 139 139 139 250 250 250 F F F 3 060-079 143 143 143 260 260 260 F F F 4 080-099 142 142 142 256 256 256 F F F 5 100-119 140 140 140 254 254 254 F F F 6 120-139 143 143 143 260 260 260 F F F 7 140-159 143 143 143 260 260 260 F F F 8 160-179 142 142 142 258 258 258 F F F 9 180-199 142 142 142 258 258 258 F F F 10 200-219 142 142 142 256 256 256 F F F 11 220-239 140 140 140 254 254 254 F F F 12 240-259 138 138 138 249 249 249 F F F 13 260-279 130 130 130 237 237 237 F F F 14 280-299 143 143 143 260 260 260 F F F 15 300-319 141 141 141 255 255 255 F F F 16 320-339 142 142 142 257 257 257 F F F 17 340-359 139 139 139 250 250 250 F F F 18 360-379 140 140 140 251 251 251 F F F 19 380-399 142 142 142 258 258 258 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 140 140 140 254 254 254 F F F 22 440-459 143 143 143 260 260 260 F F F 23 460-479 135 135 135 246 246 246 F F F 24 480-499 127 127 127 231 231 231 F F F 25 500-519 139 139 139 250 250 250 F F F 26 520-539 142 142 142 256 256 256 F F F 27 540-559 141 141 141 255 255 255 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 142 142 142 259 259 259 F F F Max gen: 143, Max flush: 260 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.798826885Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.842561976Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.843230241Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.843280526Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.843299445Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.843317453Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763} {"msg":"Crucible 138c697e-aa96-4e8f-9120-4e6b0399af02 has session id: 3b1e534d-8b1e-4f54-b6d2-139f85608972","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.843920354Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Upstairs opts: Upstairs UUID: 138c697e-aa96-4e8f-9120-4e6b0399af02, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.84396071Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Crucible stats registered with UUID: 138c697e-aa96-4e8f-9120-4e6b0399af02","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.843980258Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:53.843993277Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844362351Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844386868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763} {"msg":"138c697e-aa96-4e8f-9120-4e6b0399af02 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844446433Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844463291Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"0","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844486079Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"0","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844499777Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"1","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844514056Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"1","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844527155Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"2","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844550632Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"2","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.84457327Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844596518Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844620276Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"ds_connection connected from Ok(127.0.0.1:50372)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844651493Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"io task","client":"2","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"ds_connection connected from Ok(127.0.0.1:36309)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844754013Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"io task","client":"0","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"ds_connection connected from Ok(127.0.0.1:51951)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.844855173Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"io task","client":"1","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.845359614Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"2","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.84540021Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"0","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.845538606Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"1","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846033948Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"2","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846061885Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"1","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846077304Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"0","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846669276Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846702653Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"[0]R flush_numbers[0..12]: [248, 249, 250, 260, 256, 254, 260, 260, 258, 258, 256, 254]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846725261Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"[0]R generation[0..12]: [138, 138, 139, 143, 142, 140, 143, 143, 142, 142, 142, 140]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846749589Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846773326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"[1]R flush_numbers[0..12]: [248, 249, 250, 260, 256, 254, 260, 260, 258, 258, 256, 254]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846802333Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"[1]R generation[0..12]: [138, 138, 139, 143, 142, 140, 143, 143, 142, 142, 142, 140]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846829871Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846854418Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"[2]R flush_numbers[0..12]: [248, 249, 250, 260, 256, 254, 260, 260, 258, 258, 256, 254]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846878666Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"[2]R generation[0..12]: [138, 138, 139, 143, 142, 140, 143, 143, 142, 142, 142, 140]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846905803Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:14:53.846933531Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846961868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.846987075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.847032741Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.847062658Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"":"downstairs","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.847090425Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.847113253Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"0","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.847139011Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"1","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Transition from WaitQuorum to Active","v":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.847168568Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"client":"2","":"downstairs","session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.847195295Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.847209774Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"138c697e-aa96-4e8f-9120-4e6b0399af02 is now active with session: 3b1e534d-8b1e-4f54-b6d2-139f85608972","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.847222592Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.847234371Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763,"session_id":"3b1e534d-8b1e-4f54-b6d2-139f85608972"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:53.84724613Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 499 len 32768 data: 11 10 9 10 14 14 15 14 02/30 Read block 152 len 24576 03/30 Write block 301 len 40960 data: 15 12 13 12 12 12 13 15 19 18 04/30 Read block 98 len 36864 05/30 Write block 95 len 12288 data: 10 11 12 06/30 Read block 406 len 16384 07/30 Write block 506 len 36864 data: 15 11 11 14 15 15 15 14 12 08/30 Read block 469 len 36864 09/30 Flush 10/30 Write block 274 len 32768 data: 11 11 12 11 13 12 8 7 11/30 Read block 595 len 12288 12/30 Write block 22 len 4096 data: 10 13/30 Read block 513 len 4096 14/30 Write block 221 len 16384 data: 9 13 14 15 15/30 Flush 16/30 Write block 450 len 40960 data: 14 11 8 9 8 9 10 9 8 10 17/30 Write block 292 len 32768 data: 11 11 12 10 12 11 12 10 18/30 Read block 174 len 16384 19/30 Write block 113 len 20480 data: 13 13 12 10 9 20/30 Flush 21/30 Write block 127 len 8192 data: 5 4 22/30 Write block 588 len 8192 data: 14 18 23/30 Read block 389 len 28672 24/30 Read block 317 len 32768 25/30 Read block 20 len 8192 26/30 Write block 594 len 12288 data: 12 8 8 27/30 Read block 334 len 12288 28/30 Read block 221 len 32768 29/30 Write block 586 len 32768 data: 12 12 15 19 17 17 16 15 30/30 Write block 209 len 12288 data: 12 14 12 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:53.866301935Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1763} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 248 248 248 F F F 1 020-039 144 144 144 262 262 262 F F F 2 040-059 139 139 139 250 250 250 F F F 3 060-079 143 143 143 260 260 260 F F F 4 080-099 144 144 144 261 261 261 F F F 5 100-119 144 144 144 263 263 263 F F F 6 120-139 143 143 143 260 260 260 T T T 7 140-159 143 143 143 260 260 260 F F F 8 160-179 142 142 142 258 258 258 F F F 9 180-199 142 142 142 258 258 258 F F F 10 200-219 142 142 142 256 256 256 T T T 11 220-239 144 144 144 262 262 262 F F F 12 240-259 138 138 138 249 249 249 F F F 13 260-279 144 144 144 262 262 262 F F F 14 280-299 144 144 144 263 263 263 F F F 15 300-319 144 144 144 261 261 261 F F F 16 320-339 142 142 142 257 257 257 F F F 17 340-359 139 139 139 250 250 250 F F F 18 360-379 140 140 140 251 251 251 F F F 19 380-399 142 142 142 258 258 258 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 140 140 140 254 254 254 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 135 135 135 246 246 246 F F F 24 480-499 144 144 144 261 261 261 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 142 142 142 256 256 256 F F F 27 540-559 141 141 141 255 255 255 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 142 142 142 259 259 259 T T T Max gen: 144, Max flush: 263 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 248 248 248 F F F 1 020-039 144 144 144 262 262 262 F F F 2 040-059 139 139 139 250 250 250 F F F 3 060-079 143 143 143 260 260 260 F F F 4 080-099 144 144 144 261 261 261 F F F 5 100-119 144 144 144 263 263 263 F F F 6 120-139 145 145 145 264 264 264 F F F 7 140-159 143 143 143 260 260 260 F F F 8 160-179 142 142 142 258 258 258 F F F 9 180-199 142 142 142 258 258 258 F F F 10 200-219 145 145 145 264 264 264 F F F 11 220-239 144 144 144 262 262 262 F F F 12 240-259 138 138 138 249 249 249 F F F 13 260-279 144 144 144 262 262 262 F F F 14 280-299 144 144 144 263 263 263 F F F 15 300-319 144 144 144 261 261 261 F F F 16 320-339 142 142 142 257 257 257 F F F 17 340-359 139 139 139 250 250 250 F F F 18 360-379 140 140 140 251 251 251 F F F 19 380-399 142 142 142 258 258 258 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 140 140 140 254 254 254 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 135 135 135 246 246 246 F F F 24 480-499 144 144 144 261 261 261 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 142 142 142 256 256 256 F F F 27 540-559 141 141 141 255 255 255 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 145 145 145 264 264 264 F F F Max gen: 145, Max flush: 264 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.087125712Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.130784991Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.131404421Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.131443167Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.131461265Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.131479113Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771} {"msg":"Crucible 28f4b7b1-c6bb-4044-b6bf-ec2d9a43deb6 has session id: c4e21f75-ddd9-42d8-a569-17b25bcbb5fc","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132057197Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Upstairs opts: Upstairs UUID: 28f4b7b1-c6bb-4044-b6bf-ec2d9a43deb6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132113532Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Crucible stats registered with UUID: 28f4b7b1-c6bb-4044-b6bf-ec2d9a43deb6","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132139119Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:58.132155528Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132627851Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132663558Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771} {"msg":"28f4b7b1-c6bb-4044-b6bf-ec2d9a43deb6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132692835Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132713213Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"0","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.13274459Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"0","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132774817Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"1","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132806544Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"1","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132833521Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"2","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132859649Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"2","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132882327Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132909204Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132936481Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:43341)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132966149Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:50164)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.132992196Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:38781)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.133203345Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.133639493Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"1","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.133679339Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"0","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.133709466Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"2","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.13418544Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"1","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.134307808Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"0","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.134345404Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"2","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.134724017Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.134842566Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"[0]R flush_numbers[0..12]: [248, 262, 250, 260, 261, 263, 264, 260, 258, 258, 264, 262]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.134973523Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"[0]R generation[0..12]: [138, 144, 139, 143, 144, 144, 145, 143, 142, 142, 145, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.134994071Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135024668Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"[1]R flush_numbers[0..12]: [248, 262, 250, 260, 261, 263, 264, 260, 258, 258, 264, 262]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135039836Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"[1]R generation[0..12]: [138, 144, 139, 143, 144, 144, 145, 143, 142, 142, 145, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135053345Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135066914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"[2]R flush_numbers[0..12]: [248, 262, 250, 260, 261, 263, 264, 260, 258, 258, 264, 262]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135080263Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"[2]R generation[0..12]: [138, 144, 139, 143, 144, 144, 145, 143, 142, 142, 145, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135093591Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.13510697Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135127018Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135140027Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135156765Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135169684Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"":"downstairs","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135182453Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135194161Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"0","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.1352069Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"1","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135219689Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"client":"2","":"downstairs","session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135233648Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135245727Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"28f4b7b1-c6bb-4044-b6bf-ec2d9a43deb6 is now active with session: c4e21f75-ddd9-42d8-a569-17b25bcbb5fc","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135258235Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135269874Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771,"session_id":"c4e21f75-ddd9-42d8-a569-17b25bcbb5fc"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:14:58.135281733Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 416 len 32768 data: 10 9 8 9 10 10 9 10 02/30 Read block 438 len 24576 03/30 Write block 202 len 12288 data: 12 14 15 04/30 Read block 386 len 32768 05/30 Read block 82 len 4096 06/30 Read block 30 len 40960 07/30 Read block 294 len 8192 08/30 Write block 158 len 40960 data: 14 12 12 13 14 16 20 16 13 13 09/30 Read block 151 len 24576 10/30 Read block 377 len 12288 11/30 Write block 422 len 16384 data: 10 11 9 9 12/30 Flush 13/30 Write block 541 len 4096 data: 14 14/30 Flush 15/30 Write block 116 len 4096 data: 11 16/30 Read block 12 len 12288 17/30 Read block 347 len 20480 18/30 Read block 333 len 28672 19/30 Read block 518 len 8192 20/30 Read block 274 len 20480 21/30 Read block 323 len 20480 22/30 Write block 293 len 8192 data: 12 13 23/30 Read block 389 len 4096 24/30 Write block 222 len 28672 data: 14 15 16 15 14 11 13 25/30 Read block 207 len 12288 26/30 Write block 12 len 16384 data: 13 14 13 14 27/30 Read block 411 len 16384 28/30 Write block 228 len 20480 data: 14 14 11 9 7 29/30 Write block 61 len 24576 data: 11 11 11 10 13 13 30/30 Read block 280 len 4096 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 1 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 14 16 16 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:14:58.154442227Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1771} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 248 248 248 T T T 1 020-039 144 144 144 262 262 262 F F F 2 040-059 139 139 139 250 250 250 F F F 3 060-079 143 143 143 260 260 260 T T T 4 080-099 144 144 144 261 261 261 F F F 5 100-119 144 144 144 263 263 263 T T T 6 120-139 145 145 145 264 264 264 F F F 7 140-159 146 146 146 265 265 265 F F F 8 160-179 146 146 146 265 265 265 F F F 9 180-199 142 142 142 258 258 258 F F F 10 200-219 146 146 146 265 265 265 F F F 11 220-239 144 144 144 262 262 262 T T T 12 240-259 138 138 138 249 249 249 F F F 13 260-279 144 144 144 262 262 262 F F F 14 280-299 144 144 144 263 263 263 T T T 15 300-319 144 144 144 261 261 261 F F F 16 320-339 142 142 142 257 257 257 F F F 17 340-359 139 139 139 250 250 250 F F F 18 360-379 140 140 140 251 251 251 F F F 19 380-399 142 142 142 258 258 258 F F F 20 400-419 146 146 146 265 265 265 F F F 21 420-439 146 146 146 265 265 265 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 135 135 135 246 246 246 F F F 24 480-499 144 144 144 261 261 261 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 142 142 142 256 256 256 F F F 27 540-559 146 146 146 266 266 266 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 145 145 145 264 264 264 F F F Max gen: 146, Max flush: 266 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 267 267 267 F F F 1 020-039 144 144 144 262 262 262 F F F 2 040-059 139 139 139 250 250 250 F F F 3 060-079 147 147 147 267 267 267 F F F 4 080-099 144 144 144 261 261 261 F F F 5 100-119 147 147 147 267 267 267 F F F 6 120-139 145 145 145 264 264 264 F F F 7 140-159 146 146 146 265 265 265 F F F 8 160-179 146 146 146 265 265 265 F F F 9 180-199 142 142 142 258 258 258 F F F 10 200-219 146 146 146 265 265 265 F F F 11 220-239 147 147 147 267 267 267 F F F 12 240-259 138 138 138 249 249 249 F F F 13 260-279 144 144 144 262 262 262 F F F 14 280-299 147 147 147 267 267 267 F F F 15 300-319 144 144 144 261 261 261 F F F 16 320-339 142 142 142 257 257 257 F F F 17 340-359 139 139 139 250 250 250 F F F 18 360-379 140 140 140 251 251 251 F F F 19 380-399 142 142 142 258 258 258 F F F 20 400-419 146 146 146 265 265 265 F F F 21 420-439 146 146 146 265 265 265 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 135 135 135 246 246 246 F F F 24 480-499 144 144 144 261 261 261 F F F 25 500-519 144 144 144 261 261 261 F F F 26 520-539 142 142 142 256 256 256 F F F 27 540-559 146 146 146 266 266 266 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 145 145 145 264 264 264 F F F Max gen: 147, Max flush: 267 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.391437809Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.434267768Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.434896497Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.434945802Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.43496379Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.434984448Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779} {"msg":"Crucible 9917dc45-cc2a-439c-b700-d0b38cad57b3 has session id: 97f6ecb7-7f4c-4be0-9295-8423a0856c7c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.435734285Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Upstairs opts: Upstairs UUID: 9917dc45-cc2a-439c-b700-d0b38cad57b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.435779011Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Crucible stats registered with UUID: 9917dc45-cc2a-439c-b700-d0b38cad57b3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.435798339Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:02.435811318Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.436189171Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.436212469Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779} {"msg":"9917dc45-cc2a-439c-b700-d0b38cad57b3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.436267743Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.436284882Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"0","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.43630714Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"0","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.436321248Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"1","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.436336717Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"1","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.436359434Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"2","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.436385372Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"2","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.43640923Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.436433527Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.436457505Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59299)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.436482782Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64119)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.43650918Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61580)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.436568084Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.437165726Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"2","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.43722544Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"1","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.437255777Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"0","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.43783992Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"1","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.437878776Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"2","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.437906094Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"0","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438435592Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438470409Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"[0]R flush_numbers[0..12]: [267, 262, 250, 267, 261, 267, 264, 265, 265, 258, 265, 267]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438493337Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"[0]R generation[0..12]: [147, 144, 139, 147, 144, 147, 145, 146, 146, 142, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438509265Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438523324Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"[1]R flush_numbers[0..12]: [267, 262, 250, 267, 261, 267, 264, 265, 265, 258, 265, 267]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438537022Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"[1]R generation[0..12]: [147, 144, 139, 147, 144, 147, 145, 146, 146, 142, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438550451Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438567649Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"[2]R flush_numbers[0..12]: [267, 262, 250, 267, 261, 267, 264, 265, 265, 258, 265, 267]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438581518Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"[2]R generation[0..12]: [147, 144, 139, 147, 144, 147, 145, 146, 146, 142, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438595137Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438608665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Max found gen is 148","v":0,"name"Wait for a query_work_queue command to finish before sending IO :"crucible","level":30,"time":"2025-01-28T17:15:02.438630653Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438655271Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438677099Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438700527Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"":"downstairs","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438723974Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-28T17:15:02.438745762Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"0","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.43876998Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"1","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438792027Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"client":"2","":"downstairs","session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438806686Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438818835Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"9917dc45-cc2a-439c-b700-d0b38cad57b3 is now active with session: 97f6ecb7-7f4c-4be0-9295-8423a0856c7c","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438831344Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438845622Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779,"session_id":"97f6ecb7-7f4c-4be0-9295-8423a0856c7c"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:02.438857681Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 533 len 24576 data: 16 11 13 10 12 12 02/30 Write block 171 len 12288 data: 12 15 15 03/30 Write block 334 len 20480 data: 11 8 9 10 9 04/30 Read block 434 len 36864 05/30 Write block 592 len 32768 data: 17 16 13 9 9 7 6 3 06/30 Flush 07/30 Write block 479 len 12288 data: 9 9 8 08/30 Read block 140 len 4096 09/30 Write block 266 len 24576 data: 16 15 12 10 10 10 10/30 Write block 516 len 24576 data: 17 16 17 18 18 16 11/30 Read block 299 len 32768 12/30 Write block 177 len 24576 data: 15 14 17 18 14 15 13/30 Write block 391 len 12288 data: 11 13 12 14/30 Read block 30 len 28672 15/30 Read block 149 len 8192 16/30 Read block 513 len 8192 17/30 Read block 507 len 4096 18/30 Read block 408 len 20480 19/30 Write block 428 len 16384 data: 12 12 14 14 20/30 Read block 233 len 16384 21/30 Read block 467 len 12288 22/30 Read block 234 len 8192 23/30 Flush 24/30 Write block 160 len 4096 data: 13 25/30 Flush 26/30 Write block 492 len 24576 data: 12 14 13 12 12 10 27/30 Write block 181 len 20480 data: 15 16 16 14 13 28/30 Write block 479 len 12288 data: 10 10 9 29/30 Write block 414 len 32768 data: 9 9 11 10 9 10 11 11 30/30 Read block 415 len 36864 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 5 4 3 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:02.457065598Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1779} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 267 267 267 F F F 1 020-039 144 144 144 262 262 262 F F F 2 040-059 139 139 139 250 250 250 F F F 3 060-079 147 147 147 267 267 267 F F F 4 080-099 144 144 144 261 261 261 F F F 5 100-119 147 147 147 267 267 267 F F F 6 120-139 145 145 145 264 264 264 F F F 7 140-159 146 146 146 265 265 265 F F F 8 160-179 148 148 148 270 270 270 F F F 9 180-199 148 148 148 269 269 269 T T T 10 200-219 146 146 146 265 265 265 F F F 11 220-239 147 147 147 267 267 267 F F F 12 240-259 138 138 138 249 249 249 F F F 13 260-279 148 148 148 269 269 269 F F F 14 280-299 147 147 147 267 267 267 F F F 15 300-319 144 144 144 261 261 261 F F F 16 320-339 148 148 148 268 268 268 F F F 17 340-359 139 139 139 250 250 250 F F F 18 360-379 140 140 140 251 251 251 F F F 19 380-399 148 148 148 269 269 269 F F F 20 400-419 146 146 146 265 265 265 T T T 21 420-439 148 148 148 269 269 269 T T T 22 440-459 144 144 144 263 263 263 F F F 23 460-479 148 148 148 269 269 269 T T T 24 480-499 148 148 148 269 269 269 T T T 25 500-519 148 148 148 269 269 269 F F F 26 520-539 148 148 148 269 269 269 F F F 27 540-559 146 146 146 266 266 266 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 148 148 148 268 268 268 F F F Max gen: 148, Max flush: 270 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 267 267 267 F F F 1 020-039 144 144 144 262 262 262 F F F 2 040-059 139 139 139 250 250 250 F F F 3 060-079 147 147 147 267 267 267 F F F 4 080-099 144 144 144 261 261 261 F F F 5 100-119 147 147 147 267 267 267 F F F 6 120-139 145 145 145 264 264 264 F F F 7 140-159 146 146 146 265 265 265 F F F 8 160-179 148 148 148 270 270 270 F F F 9 180-199 149 149 149 271 271 271 F F F 10 200-219 146 146 146 265 265 265 F F F 11 220-239 147 147 147 267 267 267 F F F 12 240-259 138 138 138 249 249 249 F F F 13 260-279 148 148 148 269 269 269 F F F 14 280-299 147 147 147 267 267 267 F F F 15 300-319 144 144 144 261 261 261 F F F 16 320-339 148 148 148 268 268 268 F F F 17 340-359 139 139 139 250 250 250 F F F 18 360-379 140 140 140 251 251 251 F F F 19 380-399 148 148 148 269 269 269 F F F 20 400-419 149 149 149 271 271 271 F F F 21 420-439 149 149 149 271 271 271 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 149 149 149 271 271 271 F F F 24 480-499 149 149 149 271 271 271 F F F 25 500-519 148 148 148 269 269 269 F F F 26 520-539 148 148 148 269 269 269 F F F 27 540-559 146 146 146 266 266 266 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 148 148 148 268 268 268 F F F Max gen: 149, Max flush: 271 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.687046453Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.730116159Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.730698632Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.730745667Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.730763196Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.730784714Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787} {"msg":"Crucible 9d8ff148-e3c8-49c8-9ea0-ab5a92050aa0 has session id: 1ce60764-a588-42a8-a671-11fa08382272","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.731294794Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"Upstairs opts: Upstairs UUID: 9d8ff148-e3c8-49c8-9ea0-ab5a92050aa0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.7313406Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"Crucible stats registered with UUID: 9d8ff148-e3c8-49c8-9ea0-ab5a92050aa0","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.731364327Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:06.731385555Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.731810714Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.731844991Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787} {"msg":"9d8ff148-e3c8-49c8-9ea0-ab5a92050aa0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.731876808Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.731890366Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"0","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.731910754Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"0","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.731924363Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"1","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.731946981Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"1","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.731972208Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"2","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.732002115Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"2","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.732035962Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.732067249Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.732093896Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"ds_connection connected from Ok(127.0.0.1:50468)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.732118034Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"ds_connection connected from Ok(127.0.0.1:55918)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.732143631Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"ds_connection connected from Ok(127.0.0.1:58186)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.732167479Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.732783189Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"1","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.732821176Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"0","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.733310998Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"0","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.733407658Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"1","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.733824188Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.733940216Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.734497732Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"2","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.734991334Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"2","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"[0]R flush_numbers[0..12]: [267, 262, 250, 267, 261, 267, 264, 265, 270, 271, 265, 267]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735495835Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"[0]R generation[0..12]: [147, 144, 139, 147, 144, 147, 145, 146, 148, 149, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735526602Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.73554256Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"[1]R flush_numbers[0..12]: [267, 262, 250, 267, 261, 267, 264, 265, 270, 271, 265, 267]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735563149Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"[1]R generation[0..12]: [147, 144, 139, 147, 144, 147, 145, 146, 148, 149, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735577277Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735591256Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"[2]R flush_numbers[0..12]: [267, 262, 250, 267, 261, 267, 264, 265, 270, 271, 265, 267]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735604814Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"[2]R generation[0..12]: [147, 144, 139, 147, 144, 147, 145, 146, 148, 149, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735618443Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735632082Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.73564459Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"Wait for a query_work_queue command to finish before sending IO } {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735667488Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735680337Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735692936Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"":"downstairs","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735705785Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735717774Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"0","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735733562Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"1","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735746561Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"client":"2","":"downstairs","session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735759489Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735771398Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"9d8ff148-e3c8-49c8-9ea0-ab5a92050aa0 is now active with session: 1ce60764-a588-42a8-a671-11fa08382272","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735783947Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735795566Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787,"session_id":"1ce60764-a588-42a8-a671-11fa08382272"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:06.735807485Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 49 len 24576 data: 8 8 11 12 10 10 03/30 Read block 464 len 24576 04/30 Read block 357 len 20480 05/30 Read block 505 len 4096 06/30 Read block 522 len 16384 07/30 Write block 433 len 8192 data: 15 18 08/30 Write block 94 len 36864 data: 9 11 12 13 10 9 8 7 7 09/30 Read block 223 len 8192 10/30 Read block 178 len 28672 11/30 Flush 12/30 Write block 110 len 24576 data: 11 9 10 14 14 13 13/30 Write block 13 len 20480 data: 15 14 15 14 9 14/30 Write block 340 len 28672 data: 10 10 13 12 14 14 16 15/30 Read block 119 len 8192 16/30 Read block 569 len 16384 17/30 Write block 86 len 32768 data: 11 12 12 14 14 13 11 12 18/30 Write block 479 len 36864 data: 11 11 10 10 9 7 8 10 14 19/30 Read block 498 len 36864 20/30 Write block 478 len 28672 data: 8 12 12 11 11 10 8 21/30 Read block 563 len 20480 22/30 Read block 399 len 16384 23/30 Write block 205 len 20480 data: 14 12 12 12 13 24/30 Write block 306 len 16384 data: 13 14 16 20 25/30 Write block 237 len 36864 data: 16 18 18 18 16 16 13 13 12 26/30 Write block 91 len 16384 data: 14 12 13 10 27/30 Write block 434 len 24576 data: 19 16 14 15 16 14 28/30 Write block 494 len 28672 data: 14 13 13 11 11 12 11 29/30 Read block 402 len 28672 30/30 Read block 159 len 24576 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 6 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 19 19 18 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:06.755100066Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1787} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 267 267 267 T T T 1 020-039 144 144 144 262 262 262 F F F 2 040-059 150 150 150 273 273 273 F F F 3 060-079 147 147 147 267 267 267 F F F 4 080-099 150 150 150 273 273 273 T T T 5 100-119 150 150 150 273 273 273 T T T 6 120-139 145 145 145 264 264 264 F F F 7 140-159 146 146 146 265 265 265 F F F 8 160-179 148 148 148 270 270 270 F F F 9 180-199 149 149 149 271 271 271 F F F 10 200-219 146 146 146 265 265 265 T T T 11 220-239 147 147 147 267 267 267 T T T 12 240-259 138 138 138 249 249 249 T T T 13 260-279 148 148 148 269 269 269 F F F 14 280-299 147 147 147 267 267 267 F F F 15 300-319 144 144 144 261 261 261 T T T 16 320-339 148 148 148 268 268 268 F F F 17 340-359 139 139 139 250 250 250 T T T 18 360-379 140 140 140 251 251 251 F F F 19 380-399 148 148 148 269 269 269 F F F 20 400-419 149 149 149 271 271 271 F F F 21 420-439 150 150 150 273 273 273 T T T 22 440-459 144 144 144 263 263 263 F F F 23 460-479 149 149 149 271 271 271 T T T 24 480-499 149 149 149 271 271 271 T T T 25 500-519 148 148 148 269 269 269 T T T 26 520-539 148 148 148 269 269 269 F F F 27 540-559 146 146 146 266 266 266 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 148 148 148 268 268 268 F F F Max gen: 150, Max flush: 273 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 274 274 274 F F F 1 020-039 144 144 144 262 262 262 F F F 2 040-059 150 150 150 273 273 273 F F F 3 060-079 147 147 147 267 267 267 F F F 4 080-099 151 151 151 274 274 274 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 145 145 145 264 264 264 F F F 7 140-159 146 146 146 265 265 265 F F F 8 160-179 148 148 148 270 270 270 F F F 9 180-199 149 149 149 271 271 271 F F F 10 200-219 151 151 151 274 274 274 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 151 151 151 274 274 274 F F F 13 260-279 148 148 148 269 269 269 F F F 14 280-299 147 147 147 267 267 267 F F F 15 300-319 151 151 151 274 274 274 F F F 16 320-339 148 148 148 268 268 268 F F F 17 340-359 151 151 151 274 274 274 F F F 18 360-379 140 140 140 251 251 251 F F F 19 380-399 148 148 148 269 269 269 F F F 20 400-419 149 149 149 271 271 271 F F F 21 420-439 151 151 151 274 274 274 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 151 151 151 274 274 274 F F F 24 480-499 151 151 151 274 274 274 F F F 25 500-519 151 151 151 274 274 274 F F F 26 520-539 148 148 148 269 269 269 F F F 27 540-559 146 146 146 266 266 266 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 148 148 148 268 268 268 F F F Max gen: 151, Max flush: 274 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.027912522Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.07095672Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.071534674Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.07157625Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.071591019Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.071605288Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795} {"msg":"Crucible e4b86f00-b7a8-4491-8452-232246b5a6b4 has session id: be3182ab-da37-4cd8-a67e-9c7eb72a74cb","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.07219783Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"Upstairs opts: Upstairs UUID: e4b86f00-b7a8-4491-8452-232246b5a6b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.072305069Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"Crucible stats registered with UUID: e4b86f00-b7a8-4491-8452-232246b5a6b4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.072331897Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:21.072346375Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.072715109Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.072740177Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795} {"msg":"e4b86f00-b7a8-4491-8452-232246b5a6b4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.072785562Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.072803471Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"0","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.072833128Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"0","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.072859585Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"1","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.072886383Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"1","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.07291424Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"2","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.072942827Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"2","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.072970775Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.072997932Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.07302139Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:59264)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.073047927Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:45565)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.073076264Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:58122)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.073105601Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.073679695Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"0","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.073724431Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"2","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.073830251Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"1","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.074332672Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"0","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.074368039Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"2","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.074396256Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"1","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.074942232Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075079179Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"[0]R flush_numbers[0..12]: [274, 262, 273, 267, 274, 274, 264, 265, 270, 271, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075150462Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"[0]R generation[0..12]: [151, 144, 150, 147, 151, 151, 145, 146, 148, 149, 151, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.07517053Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075185319Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"[1]R flush_numbers[0..12]: [274, 262, 273, 267, 274, 274, 264, 265, 270, 271, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075199238Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"[1]R generation[0..12]: [151, 144, 150, 147, 151, 151, 145, 146, 148, 149, 151, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075213436Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075227445Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"[2]R flush_numbers[0..12]: [274, 262, 273, 267, 274, 274, 264, 265, 270, 271, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075241083Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"[2]R generation[0..12]: [151, 144, 150, 147, 151, 151, 145, 146, 148, 149, 151, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075258922Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.0752733Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075291429Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075306137Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075329195Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075347863Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"":"downstairs","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075361342Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.07537329Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"0","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075386559Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"1","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075399418Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"client":"2","":"downstairs","session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075412277Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075431555Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"e4b86f00-b7a8-4491-8452-232246b5a6b4 is now active with session: be3182ab-da37-4cd8-a67e-9c7eb72a74cb","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075453433Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.075466212Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795,"session_id":"be3182ab-da37-4cd8-a67e-9c7eb72a74cb"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:21.07548134Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 511 len 28672 data: 16 16 15 13 13 18 17 02/30 Flush 03/30 Write block 64 len 4096 data: 11 04/30 Read block 152 len 40960 05/30 Read block 217 len 4096 06/30 Write block 38 len 40960 data: 10 9 10 9 9 10 10 8 8 10 07/30 Write block 358 len 20480 data: 15 14 11 10 7 08/30 Flush 09/30 Write block 364 len 12288 data: 8 8 7 10/30 Read block 81 len 16384 11/30 Write block 308 len 12288 data: 17 21 19 12/30 Read block 49 len 16384 13/30 Read block 337 len 40960 14/30 Read block 582 len 4096 15/30 Flush 16/30 Flush 17/30 Write block 214 len 8192 data: 12 11 18/30 Read block 280 len 28672 19/30 Write block 332 len 16384 data: 8 11 12 9 20/30 Read block 350 len 12288 21/30 Write block 283 len 20480 data: 6 5 7 7 8 22/30 Read block 531 len 28672 23/30 Write block 498 len 32768 data: 12 13 12 10 11 15 15 16 24/30 Read block 66 len 16384 25/30 Read block 413 len 40960 26/30 Read block 229 len 20480 27/30 Read block 243 len 28672 28/30 Write block 210 len 12288 data: 15 13 10 29/30 Write block 55 len 20480 data: 9 6 7 7 7 30/30 Read block 223 len 32768 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 2 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 14 12 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:21.094729946Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1795} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 274 274 274 F F F 1 020-039 152 152 152 276 276 276 F F F 2 040-059 152 152 152 276 276 276 T T T 3 060-079 152 152 152 276 276 276 F F F 4 080-099 151 151 151 274 274 274 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 145 145 145 264 264 264 F F F 7 140-159 146 146 146 265 265 265 F F F 8 160-179 148 148 148 270 270 270 F F F 9 180-199 149 149 149 271 271 271 F F F 10 200-219 151 151 151 274 274 274 T T T 11 220-239 151 151 151 274 274 274 F F F 12 240-259 151 151 151 274 274 274 F F F 13 260-279 148 148 148 269 269 269 F F F 14 280-299 147 147 147 267 267 267 T T T 15 300-319 152 152 152 277 277 277 F F F 16 320-339 148 148 148 268 268 268 T T T 17 340-359 152 152 152 276 276 276 F F F 18 360-379 152 152 152 277 277 277 F F F 19 380-399 148 148 148 269 269 269 F F F 20 400-419 149 149 149 271 271 271 F F F 21 420-439 151 151 151 274 274 274 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 151 151 151 274 274 274 F F F 24 480-499 151 151 151 274 274 274 T T T 25 500-519 152 152 152 275 275 275 T T T 26 520-539 148 148 148 269 269 269 F F F 27 540-559 146 146 146 266 266 266 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 148 148 148 268 268 268 F F F Max gen: 152, Max flush: 277 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 274 274 274 F F F 1 020-039 152 152 152 276 276 276 F F F 2 040-059 153 153 153 278 278 278 F F F 3 060-079 152 152 152 276 276 276 F F F 4 080-099 151 151 151 274 274 274 F F F 5 100-119 151 151 151 274 274 274 F F F 6 120-139 145 145 145 264 264 264 F F F 7 140-159 146 146 146 265 265 265 F F F 8 160-179 148 148 148 270 270 270 F F F 9 180-199 149 149 149 271 271 271 F F F 10 200-219 153 153 153 278 278 278 F F F 11 220-239 151 151 151 274 274 274 F F F 12 240-259 151 151 151 274 274 274 F F F 13 260-279 148 148 148 269 269 269 F F F 14 280-299 153 153 153 278 278 278 F F F 15 300-319 152 152 152 277 277 277 F F F 16 320-339 153 153 153 278 278 278 F F F 17 340-359 152 152 152 276 276 276 F F F 18 360-379 152 152 152 277 277 277 F F F 19 380-399 148 148 148 269 269 269 F F F 20 400-419 149 149 149 271 271 271 F F F 21 420-439 151 151 151 274 274 274 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 151 151 151 274 274 274 F F F 24 480-499 153 153 153 278 278 278 F F F 25 500-519 153 153 153 278 278 278 F F F 26 520-539 148 148 148 269 269 269 F F F 27 540-559 146 146 146 266 266 266 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 148 148 148 268 268 268 F F F Max gen: 153, Max flush: 278 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.332078054Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.375830133Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.376489199Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.376530055Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.376545533Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.376559582Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803} {"msg":"Crucible 3612f25e-bb6b-42f5-8b59-f61b0db1ee06 has session id: 1d6853e4-656c-4c07-a210-6c6d93b5acf0","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.377164223Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Upstairs opts: Upstairs UUID: 3612f25e-bb6b-42f5-8b59-f61b0db1ee06, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.377212289Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Crucible stats registered with UUID: 3612f25e-bb6b-42f5-8b59-f61b0db1ee06","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.377231547Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:25.377245065Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.377799251Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.377829479Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803} {"msg":"3612f25e-bb6b-42f5-8b59-f61b0db1ee06 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.377897922Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.377932269Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"0","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.377964795Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"0","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.377987013Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"1","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.378012071Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"1","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.378035668Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"2","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.378060816Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"2","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.378082904Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.378108761Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.378133479Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36992)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.378158887Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36131)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.378185984Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"ds_connection connected from Ok(127.0.0.1:63378)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.378213921Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.378849949Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"2","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.378887925Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"0","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.379032401Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"1","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.379469399Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"2","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.379509125Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"0","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.379539972Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"1","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.38006991Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380110186Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"[0]R flush_numbers[0..12]: [274, 276, 278, 276, 274, 274, 264, 265, 270, 271, 278, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380131024Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"[0]R generation[0..12]: [151, 152, 153, 152, 151, 151, 145, 146, 148, 149, 153, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380150293Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380165341Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"[1]R flush_numbers[0..12]: [274, 276, 278, 276, 274, 274, 264, 265, 270, 271, 278, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380181259Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"[1]R generation[0..12]: [151, 152, 153, 152, 151, 151, 145, 146, 148, 149, 153, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380206137Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380232594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"[2]R flush_numbers[0..12]: [274, 276, 278, 276, 274, 274, 264, 265, 270, 271, 278, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380256122Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"[2]R generation[0..12]: [151, 152, 153, 152, 151, 151, 145, 146, 148, 149, 153, 151]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:15:25.38028014Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380309147Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380327665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380341094Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380362412Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.38038643Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"":"downstairs","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380411647Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380425316Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"0","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380441854Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"1","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380468542Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"client":"2","":"downstairs","session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380494379Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380514137Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"3612f25e-bb6b-42f5-8b59-f61b0db1ee06 is now active with session: 1d6853e4-656c-4c07-a210-6c6d93b5acf0","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380528046Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803,"session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380546604Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :1803,"session_id":"1d6853e4-656c-4c07-a210-6c6d93b5acf0"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:25.380571141Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 266 len 40960 data: 17 16 13 11 11 11 8 8 12 12 02/30 Flush 03/30 Write block 155 len 8192 data: 9 8 04/30 Read block 440 len 8192 05/30 Flush 06/30 Write block 30 len 20480 data: 10 10 9 9 10 07/30 Write block 219 len 32768 data: 8 9 10 15 16 17 16 15 08/30 Read block 77 len 40960 09/30 Write block 492 len 20480 data: 13 15 15 14 14 10/30 Read block 410 len 40960 11/30 Write block 135 len 32768 data: 10 10 14 15 16 20 17 14 12/30 Write block 376 len 8192 data: 9 8 13/30 Read block 354 len 20480 14/30 Write block 173 len 16384 data: 16 15 16 15 15/30 Read block 312 len 16384 16/30 Read block 566 len 28672 17/30 Read block 245 len 28672 18/30 Read block 173 len 16384 19/30 Write block 215 len 28672 data: 12 12 9 8 9 10 11 20/30 Read block 55 len 12288 21/30 Write block 588 len 8192 data: 16 20 22/30 Read block 387 len 4096 23/30 Read block 404 len 36864 24/30 Read block 172 len 4096 25/30 Write block 99 len 8192 data: 10 9 26/30 Read block 303 len 36864 27/30 Read block 595 len 20480 28/30 Write block 54 len 4096 data: 11 29/30 Read block 225 len 16384 30/30 Write block 5 len 32768 data: 8 7 10 11 12 12 13 14 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 5 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:25.399284859Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1803} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 274 274 274 T T T 1 020-039 152 152 152 276 276 276 T T T 2 040-059 153 153 153 278 278 278 T T T 3 060-079 152 152 152 276 276 276 F F F 4 080-099 151 151 151 274 274 274 T T T 5 100-119 151 151 151 274 274 274 T T T 6 120-139 145 145 145 264 264 264 T T T 7 140-159 154 154 154 280 280 280 T T T 8 160-179 148 148 148 270 270 270 T T T 9 180-199 149 149 149 271 271 271 F F F 10 200-219 153 153 153 278 278 278 T T T 11 220-239 151 151 151 274 274 274 T T T 12 240-259 151 151 151 274 274 274 F F F 13 260-279 154 154 154 279 279 279 F F F 14 280-299 153 153 153 278 278 278 F F F 15 300-319 152 152 152 277 277 277 F F F 16 320-339 153 153 153 278 278 278 F F F 17 340-359 152 152 152 276 276 276 F F F 18 360-379 152 152 152 277 277 277 T T T 19 380-399 148 148 148 269 269 269 F F F 20 400-419 149 149 149 271 271 271 F F F 21 420-439 151 151 151 274 274 274 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 151 151 151 274 274 274 F F F 24 480-499 153 153 153 278 278 278 T T T 25 500-519 153 153 153 278 278 278 F F F 26 520-539 148 148 148 269 269 269 F F F 27 540-559 146 146 146 266 266 266 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 148 148 148 268 268 268 T T T Max gen: 154, Max flush: 280 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 281 281 281 F F F 1 020-039 155 155 155 281 281 281 F F F 2 040-059 155 155 155 281 281 281 F F F 3 060-079 152 152 152 276 276 276 F F F 4 080-099 155 155 155 281 281 281 F F F 5 100-119 155 155 155 281 281 281 F F F 6 120-139 155 155 155 281 281 281 F F F 7 140-159 155 155 155 281 281 281 F F F 8 160-179 155 155 155 281 281 281 F F F 9 180-199 149 149 149 271 271 271 F F F 10 200-219 155 155 155 281 281 281 F F F 11 220-239 155 155 155 281 281 281 F F F 12 240-259 151 151 151 274 274 274 F F F 13 260-279 154 154 154 279 279 279 F F F 14 280-299 153 153 153 278 278 278 F F F 15 300-319 152 152 152 277 277 277 F F F 16 320-339 153 153 153 278 278 278 F F F 17 340-359 152 152 152 276 276 276 F F F 18 360-379 155 155 155 281 281 281 F F F 19 380-399 148 148 148 269 269 269 F F F 20 400-419 149 149 149 271 271 271 F F F 21 420-439 151 151 151 274 274 274 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 151 151 151 274 274 274 F F F 24 480-499 155 155 155 281 281 281 F F F 25 500-519 153 153 153 278 278 278 F F F 26 520-539 148 148 148 269 269 269 F F F 27 540-559 146 146 146 266 266 266 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 155 155 155 281 281 281 F F F Max gen: 155, Max flush: 281 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.676209546Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:39.71806344Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.718437254Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.749443035Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.750033017Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.750068284Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.750083242Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.750097631Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811} {"msg":"Crucible e78487d3-b869-4186-8d6c-50150cd0b830 has session id: f252363e-79a3-45da-93de-020f4e5a7ba3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.75071936Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Upstairs opts: Upstairs UUID: e78487d3-b869-4186-8d6c-50150cd0b830, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.750765436Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Crucible stats registered with UUID: e78487d3-b869-4186-8d6c-50150cd0b830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.750789864Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:39.750810071Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751196484Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751222991Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811} {"msg":"e78487d3-b869-4186-8d6c-50150cd0b830 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751263827Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751280506Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"0","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751302874Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"0","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751316752Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"1","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751330631Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"1","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.75134343Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"2","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751357178Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"2","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751417423Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751457879Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751487676Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43580)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751513613Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"ds_connection connected from Ok(127.0.0.1:64718)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.7515392Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"ds_connection connected from Ok(127.0.0.1:51396)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.751571857Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.75226914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"1","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.752306146Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"2","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.752336253Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"0","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753068212Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"2","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753095989Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"1","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753110818Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"0","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753648895Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753682982Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"[0]R flush_numbers[0..12]: [281, 281, 281, 276, 281, 281, 281, 281, 281, 271, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.75370706Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"[0]R generation[0..12]: [155, 155, 155, 152, 155, 155, 155, 155, 155, 149, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753724278Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753738326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"[1]R flush_numbers[0..12]: [281, 281, 281, 276, 281, 281, 281, 281, 281, 271, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753751745Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"[1]R generation[0..12]: [155, 155, 155, 152, 155, 155, 155, 155, 155, 149, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753765274Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753782822Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"[2]R flush_numbers[0..12]: [281, 281, 281, 276, 281, 281, 281, 281, 281, 271, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753797031Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"[2]R generation[0..12]: [155, 155, 155, 152, 155, 155, 155, 155, 155, 149, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753810639Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753828868Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753851266Z",Wait for a query_work_queue command to finish before sending IO "hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753880883Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753897481Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.75391015Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"":"downstairs","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753923119Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753934987Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"0","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753948066Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"1","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753960765Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"client":"2","":"downstairs","session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753973603Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.753985572Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"e78487d3-b869-4186-8d6c-50150cd0b830 is now active with session: f252363e-79a3-45da-93de-020f4e5a7ba3","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-28T17:15:39.753998071Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.75401307Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811,"session_id":"f252363e-79a3-45da-93de-020f4e5a7ba3"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:39.754025478Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 48 len 40960 data: 8 9 9 12 13 11 12 10 7 8 03/30 Write block 425 len 4096 data: 10 04/30 Flush 05/30 Flush 06/30 Flush 07/30 Write block 477 len 28672 data: 6 9 13 13 12 12 11 08/30 Write block 278 len 24576 data: 14 13 9 8 8 7 09/30 Read block 553 len 12288 10/30 Read block 417 len 12288 11/30 Write block 241 len 8192 data: 17 17 12/30 Write block 371 len 28672 data: 9 9 6 7 8 10 9 13/30 Write block 401 len 4096 data: 15 14/30 Write block 194 len 12288 data: 12 13 13 15/30 Read block 369 len 20480 16/30 Write block 391 len 16384 data: 12 14 13 9 17/30 Read block 447 len 40960 18/30 Write block 582 len 24576 data: 19 19 20 22 13 13 19/30 Flush 20/30 Write block 285 len 4096 data: 8 21/30 Read block 262 len 12288 22/30 Write block 24 len 32768 data: 11 12 12 11 11 11 11 11 23/30 Write block 541 len 28672 data: 15 11 10 10 13 13 10 24/30 Write block 537 len 28672 data: 13 13 13 15 16 12 11 25/30 Write block 149 len 12288 data: 9 10 8 26/30 Read block 444 len 12288 27/30 Read block 298 len 28672 28/30 Read block 572 len 20480 29/30 Write block 171 len 32768 data: 13 16 17 16 17 16 16 15 30/30 Read block 252 len 20480 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 1 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:39.772268422Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1811} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 281 281 281 F F F 1 020-039 155 155 155 281 281 281 T T T 2 040-059 156 156 156 283 283 283 F F F 3 060-079 152 152 152 276 276 276 F F F 4 080-099 155 155 155 281 281 281 F F F 5 100-119 155 155 155 281 281 281 F F F 6 120-139 155 155 155 281 281 281 F F F 7 140-159 155 155 155 281 281 281 T T T 8 160-179 155 155 155 281 281 281 T T T 9 180-199 156 156 156 286 286 286 F F F 10 200-219 155 155 155 281 281 281 F F F 11 220-239 155 155 155 281 281 281 F F F 12 240-259 156 156 156 286 286 286 F F F 13 260-279 156 156 156 286 286 286 F F F 14 280-299 156 156 156 286 286 286 T T T 15 300-319 152 152 152 277 277 277 F F F 16 320-339 153 153 153 278 278 278 F F F 17 340-359 152 152 152 276 276 276 F F F 18 360-379 156 156 156 286 286 286 F F F 19 380-399 156 156 156 286 286 286 F F F 20 400-419 156 156 156 286 286 286 F F F 21 420-439 156 156 156 283 283 283 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 156 156 156 286 286 286 F F F 24 480-499 156 156 156 286 286 286 F F F 25 500-519 153 153 153 278 278 278 F F F 26 520-539 148 148 148 269 269 269 T T T 27 540-559 146 146 146 266 266 266 T T T 28 560-579 142 142 142 257 257 257 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 156, Max flush: 286 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 281 281 281 F F F 1 020-039 157 157 157 287 287 287 F F F 2 040-059 156 156 156 283 283 283 F F F 3 060-079 152 152 152 276 276 276 F F F 4 080-099 155 155 155 281 281 281 F F F 5 100-119 155 155 155 281 281 281 F F F 6 120-139 155 155 155 281 281 281 F F F 7 140-159 157 157 157 287 287 287 F F F 8 160-179 157 157 157 287 287 287 F F F 9 180-199 156 156 156 286 286 286 F F F 10 200-219 155 155 155 281 281 281 F F F 11 220-239 155 155 155 281 281 281 F F F 12 240-259 156 156 156 286 286 286 F F F 13 260-279 156 156 156 286 286 286 F F F 14 280-299 157 157 157 287 287 287 F F F 15 300-319 152 152 152 277 277 277 F F F 16 320-339 153 153 153 278 278 278 F F F 17 340-359 152 152 152 276 276 276 F F F 18 360-379 156 156 156 286 286 286 F F F 19 380-399 156 156 156 286 286 286 F F F 20 400-419 156 156 156 286 286 286 F F F 21 420-439 156 156 156 283 283 283 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 156 156 156 286 286 286 F F F 24 480-499 156 156 156 286 286 286 F F F 25 500-519 153 153 153 278 278 278 F F F 26 520-539 157 157 157 287 287 287 F F F 27 540-559 157 157 157 287 287 287 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 157, Max flush: 287 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.041881857Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.085471703Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.086062415Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.08611769Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.086150477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.086174255Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819} {"msg":"Crucible 7afa5510-db85-4851-9942-4ac50285a05a has session id: eeb53814-d240-44b5-ab3f-ce8cb026bebb","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.086712292Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Upstairs opts: Upstairs UUID: 7afa5510-db85-4851-9942-4ac50285a05a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.086755848Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Crucible stats registered with UUID: 7afa5510-db85-4851-9942-4ac50285a05a","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.086772336Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:44.086785025Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.087328602Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.087362139Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819} {"msg":"7afa5510-db85-4851-9942-4ac50285a05a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.087439492Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.08745788Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"0","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.087478988Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"0","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.087492766Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"1","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.087506875Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"1","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.087523113Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"2","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.08755291Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"2","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.087578088Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.087603905Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.087630083Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43452)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.08766158Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"ds_connection connected from Ok(127.0.0.1:36404)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.087897277Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"ds_connection connected from Ok(127.0.0.1:33141)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.087936223Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.088402458Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"2","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.088430565Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"0","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.088874712Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"1","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.089049105Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"0","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.089081072Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"2","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.089535957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"1","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.089655925Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.089675404Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"[0]R flush_numbers[0..12]: [281, 287, 283, 276, 281, 281, 281, 287, 287, 286, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.089983584Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"[0]R generation[0..12]: [155, 157, 156, 152, 155, 155, 155, 157, 157, 156, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090003442Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.09001768Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"[1]R flush_numbers[0..12]: [281, 287, 283, 276, 281, 281, 281, 287, 287, 286, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090031919Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"[1]R generation[0..12]: [155, 157, 156, 152, 155, 155, 155, 157, 157, 156, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090045618Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090060026Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"[2]R flush_numbers[0..12]: [281, 287, 283, 276, 281, 281, 281, 287, 287, 286, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090073795Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"[2]R generation[0..12]: [155, 157, 156, 152, 155, 155, 155, 157, 157, 156, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090087254Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090100852Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090113461Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.09012658Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Wait for a query_work_queue command to finish before sending IO Next flush: 288","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090145288Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090158167Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"":"downstairs","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090171015Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090187034Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"0","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090200812Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"1","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090213631Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"client":"2","":"downstairs","session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.09022645Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090238419Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"7afa5510-db85-4851-9942-4ac50285a05a is now active with session: eeb53814-d240-44b5-ab3f-ce8cb026bebb","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090251118Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090262906Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819,"session_id":"eeb53814-d240-44b5-ab3f-ce8cb026bebb"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:44.090274785Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 375 len 24576 data: 9 11 10 11 10 9 02/30 Flush 03/30 Write block 206 len 32768 data: 13 13 13 14 16 14 11 12 04/30 Read block 113 len 28672 05/30 Write block 537 len 12288 data: 14 14 14 06/30 Read block 13 len 28672 07/30 Read block 12 len 16384 08/30 Read block 352 len 40960 09/30 Write block 265 len 4096 data: 16 10/30 Write block 343 len 4096 data: 13 11/30 Flush 12/30 Write block 306 len 12288 data: 14 15 18 13/30 Write block 334 len 40960 data: 13 10 10 11 10 11 11 11 14 14 14/30 Flush 15/30 Write block 464 len 32768 data: 14 15 11 10 8 9 10 10 16/30 Read block 434 len 20480 17/30 Read block 486 len 12288 18/30 Read block 387 len 8192 19/30 Read block 479 len 40960 20/30 Write block 246 len 4096 data: 10 21/30 Write block 242 len 24576 data: 18 14 14 13 11 13 22/30 Read block 189 len 40960 23/30 Flush 24/30 Write block 131 len 8192 data: 11 12 25/30 Write block 1 len 40960 data: 3 3 4 5 9 8 11 12 13 13 26/30 Write block 158 len 28672 data: 15 13 14 14 15 17 21 27/30 Write block 380 len 28672 data: 10 10 12 13 13 11 11 28/30 Write block 96 len 36864 data: 13 14 11 11 10 8 8 6 9 29/30 Write block 271 len 12288 data: 12 9 9 30/30 Read block 68 len 16384 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:44.109462437Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1819} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 155 155 155 281 281 281 T T T 1 020-039 157 157 157 287 287 287 F F F 2 040-059 156 156 156 283 283 283 F F F 3 060-079 152 152 152 276 276 276 F F F 4 080-099 155 155 155 281 281 281 T T T 5 100-119 155 155 155 281 281 281 T T T 6 120-139 155 155 155 281 281 281 T T T 7 140-159 157 157 157 287 287 287 T T T 8 160-179 157 157 157 287 287 287 T T T 9 180-199 156 156 156 286 286 286 F F F 10 200-219 158 158 158 289 289 289 F F F 11 220-239 155 155 155 281 281 281 F F F 12 240-259 158 158 158 291 291 291 F F F 13 260-279 158 158 158 289 289 289 T T T 14 280-299 157 157 157 287 287 287 F F F 15 300-319 158 158 158 290 290 290 F F F 16 320-339 158 158 158 290 290 290 F F F 17 340-359 158 158 158 290 290 290 F F F 18 360-379 158 158 158 288 288 288 F F F 19 380-399 158 158 158 288 288 288 T T T 20 400-419 156 156 156 286 286 286 F F F 21 420-439 156 156 156 283 283 283 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 158 158 158 291 291 291 F F F 24 480-499 156 156 156 286 286 286 F F F 25 500-519 153 153 153 278 278 278 F F F 26 520-539 158 158 158 289 289 289 F F F 27 540-559 157 157 157 287 287 287 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 158, Max flush: 291 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 292 292 292 F F F 1 020-039 157 157 157 287 287 287 F F F 2 040-059 156 156 156 283 283 283 F F F 3 060-079 152 152 152 276 276 276 F F F 4 080-099 159 159 159 292 292 292 F F F 5 100-119 159 159 159 292 292 292 F F F 6 120-139 159 159 159 292 292 292 F F F 7 140-159 159 159 159 292 292 292 F F F 8 160-179 159 159 159 292 292 292 F F F 9 180-199 156 156 156 286 286 286 F F F 10 200-219 158 158 158 289 289 289 F F F 11 220-239 155 155 155 281 281 281 F F F 12 240-259 158 158 158 291 291 291 F F F 13 260-279 159 159 159 292 292 292 F F F 14 280-299 157 157 157 287 287 287 F F F 15 300-319 158 158 158 290 290 290 F F F 16 320-339 158 158 158 290 290 290 F F F 17 340-359 158 158 158 290 290 290 F F F 18 360-379 158 158 158 288 288 288 F F F 19 380-399 159 159 159 292 292 292 F F F 20 400-419 156 156 156 286 286 286 F F F 21 420-439 156 156 156 283 283 283 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 158 158 158 291 291 291 F F F 24 480-499 156 156 156 286 286 286 F F F 25 500-519 153 153 153 278 278 278 F F F 26 520-539 158 158 158 289 289 289 F F F 27 540-559 157 157 157 287 287 287 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 159, Max flush: 292 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.361273586Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.404428635Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.405081871Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.405123227Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.405137995Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.405152394Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827} {"msg":"Crucible ff36dd9c-83c1-4bf0-a21e-56836380e67f has session id: 233d4df8-5d8e-4970-8ed4-a805d90e6b48","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.405799151Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Upstairs opts: Upstairs UUID: ff36dd9c-83c1-4bf0-a21e-56836380e67f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.405835967Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Crucible stats registered with UUID: ff36dd9c-83c1-4bf0-a21e-56836380e67f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.405852056Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:48.405864795Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406378425Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406402822Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827} {"msg":"ff36dd9c-83c1-4bf0-a21e-56836380e67f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406468126Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406489794Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"0","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406512512Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"0","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.40652653Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"1","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406542649Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"1","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406556077Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"2","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406569816Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"2","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406592994Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406619721Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406645559Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"ds_connection connected from Ok(127.0.0.1:49819)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406710002Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"ds_connection connected from Ok(127.0.0.1:46645)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406749978Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"ds_connection connected from Ok(127.0.0.1:59184)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.406904903Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.407381517Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"2","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.407421093Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"0","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.407526963Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"1","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408016165Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"0","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408048212Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"2","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408114405Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"1","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408643394Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408678281Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"[0]R flush_numbers[0..12]: [292, 287, 283, 276, 292, 292, 292, 292, 292, 286, 289, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408702858Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"[0]R generation[0..12]: [159, 157, 156, 152, 159, 159, 159, 159, 159, 156, 158, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408719817Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408734495Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"[1]R flush_numbers[0..12]: [292, 287, 283, 276, 292, 292, 292, 292, 292, 286, 289, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408748294Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"[1]R generation[0..12]: [159, 157, 156, 152, 159, 159, 159, 159, 159, 156, 158, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408762122Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408775981Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"[2]R flush_numbers[0..12]: [292, 287, 283, 276, 292, 292, 292, 292, 292, 286, 289, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.40878934Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"[2]R generation[0..12]: [159, 157, 156, 152, 159, 159, 159, 159, 159, 156, 158, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408802818Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408825996Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:15:48.408851364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408875922Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.40889229Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408905279Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"":"downstairs","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408918177Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408929806Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"0","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408942635Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"1","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408955454Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"client":"2","":"downstairs","session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408968333Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.408980201Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"ff36dd9c-83c1-4bf0-a21e-56836380e67f is now active with session: 233d4df8-5d8e-4970-8ed4-a805d90e6b48","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.40899263Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.409004269Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827,"session_id":"233d4df8-5d8e-4970-8ed4-a805d90e6b48"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:48.409016198Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 81 len 8192 data: 12 12 02/30 Read block 171 len 12288 03/30 Write block 72 len 28672 data: 16 15 14 13 11 10 11 04/30 Write block 335 len 20480 data: 11 11 12 11 12 05/30 Read block 432 len 12288 06/30 Write block 128 len 16384 data: 5 5 5 12 07/30 Write block 241 len 32768 data: 18 19 15 15 14 12 14 14 08/30 Read block 32 len 28672 09/30 Read block 164 len 16384 10/30 Write block 66 len 16384 data: 14 14 13 14 11/30 Read block 569 len 40960 12/30 Write block 14 len 4096 data: 15 13/30 Write block 474 len 8192 data: 6 6 14/30 Flush 15/30 Write block 364 len 12288 data: 9 9 8 16/30 Write block 558 len 8192 data: 15 12 17/30 Write block 32 len 20480 data: 10 10 11 9 8 18/30 Read block 539 len 8192 19/30 Read block 39 len 4096 20/30 Write block 15 len 36864 data: 16 15 10 8 11 10 8 11 11 21/30 Read block 505 len 4096 22/30 Read block 111 len 28672 23/30 Write block 294 len 4096 data: 14 24/30 Write block 337 len 4096 data: 13 25/30 Write block 114 len 24576 data: 15 14 12 10 7 8 26/30 Flush 27/30 Write block 138 len 16384 data: 16 17 21 18 28/30 Write block 408 len 12288 data: 15 13 13 29/30 Write block 17 len 8192 data: 11 9 30/30 Read block 544 len 40960 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 4 2 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:48.425707343Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1827} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 294 294 294 T T T 1 020-039 160 160 160 294 294 294 F F F 2 040-059 156 156 156 283 283 283 F F F 3 060-079 160 160 160 293 293 293 F F F 4 080-099 160 160 160 293 293 293 F F F 5 100-119 160 160 160 294 294 294 F F F 6 120-139 160 160 160 293 293 293 T T T 7 140-159 159 159 159 292 292 292 T T T 8 160-179 159 159 159 292 292 292 F F F 9 180-199 156 156 156 286 286 286 F F F 10 200-219 158 158 158 289 289 289 F F F 11 220-239 155 155 155 281 281 281 F F F 12 240-259 160 160 160 293 293 293 F F F 13 260-279 159 159 159 292 292 292 F F F 14 280-299 160 160 160 294 294 294 F F F 15 300-319 158 158 158 290 290 290 F F F 16 320-339 160 160 160 294 294 294 F F F 17 340-359 158 158 158 290 290 290 F F F 18 360-379 160 160 160 294 294 294 F F F 19 380-399 159 159 159 292 292 292 F F F 20 400-419 156 156 156 286 286 286 T T T 21 420-439 156 156 156 283 283 283 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 160 160 160 293 293 293 F F F 24 480-499 156 156 156 286 286 286 F F F 25 500-519 153 153 153 278 278 278 F F F 26 520-539 158 158 158 289 289 289 F F F 27 540-559 160 160 160 294 294 294 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 160, Max flush: 294 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 295 295 295 F F F 1 020-039 160 160 160 294 294 294 F F F 2 040-059 156 156 156 283 283 283 F F F 3 060-079 160 160 160 293 293 293 F F F 4 080-099 160 160 160 293 293 293 F F F 5 100-119 160 160 160 294 294 294 F F F 6 120-139 161 161 161 295 295 295 F F F 7 140-159 161 161 161 295 295 295 F F F 8 160-179 159 159 159 292 292 292 F F F 9 180-199 156 156 156 286 286 286 F F F 10 200-219 158 158 158 289 289 289 F F F 11 220-239 155 155 155 281 281 281 F F F 12 240-259 160 160 160 293 293 293 F F F 13 260-279 159 159 159 292 292 292 F F F 14 280-299 160 160 160 294 294 294 F F F 15 300-319 158 158 158 290 290 290 F F F 16 320-339 160 160 160 294 294 294 F F F 17 340-359 158 158 158 290 290 290 F F F 18 360-379 160 160 160 294 294 294 F F F 19 380-399 159 159 159 292 292 292 F F F 20 400-419 161 161 161 295 295 295 F F F 21 420-439 156 156 156 283 283 283 F F F 22 440-459 144 144 144 263 263 263 F F F 23 460-479 160 160 160 293 293 293 F F F 24 480-499 156 156 156 286 286 286 F F F 25 500-519 153 153 153 278 278 278 F F F 26 520-539 158 158 158 289 289 289 F F F 27 540-559 160 160 160 294 294 294 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 161, Max flush: 295 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.661171213Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.704373506Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.704979318Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.70505369Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.705076778Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.705096166Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835} {"msg":"Crucible d225bb39-e12a-4dc0-8310-ed2f510d09dd has session id: c0096fe6-f9b8-4666-ae4d-93214c7516b4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.705630684Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"Upstairs opts: Upstairs UUID: d225bb39-e12a-4dc0-8310-ed2f510d09dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.70566962Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"Crucible stats registered with UUID: d225bb39-e12a-4dc0-8310-ed2f510d09dd","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.705695458Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:52.705718846Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.70618662Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706211898Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835} {"msg":"d225bb39-e12a-4dc0-8310-ed2f510d09dd active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706292779Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706309938Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"0","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706332696Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"0","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706346674Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"1","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706360763Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"1","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706378751Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"2","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706403869Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"2","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706443525Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706472562Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.70649839Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:56454)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706526377Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49915)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706557744Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:54097)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.706585491Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.707243367Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"1","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.707270754Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"0","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.707920341Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"1","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.707945878Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"0","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.708504154Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.708595595Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.708924853Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"2","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.709398127Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"2","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"[0]R flush_numbers[0..12]: [295, 294, 283, 293, 293, 294, 295, 295, 292, 286, 289, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.709865022Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"[0]R generation[0..12]: [161, 160, 156, 160, 160, 160, 161, 161, 159, 156, 158, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.70988603Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.709900838Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"[1]R flush_numbers[0..12]: [295, 294, 283, 293, 293, 294, 295, 295, 292, 286, 289, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.709914517Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"[1]R generation[0..12]: [161, 160, 156, 160, 160, 160, 161, 161, 159, 156, 158, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.709932555Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.709946734Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"[2]R flush_numbers[0..12]: [295, 294, 283, 293, 293, 294, 295, 295, 292, 286, 289, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.709960153Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"[2]R generation[0..12]: [161, 160, 156, 160, 160, 160, 161, 161, 159, 156, 158, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.709973651Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.70998718Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.710007708Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.710020876Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"Next flush: 296","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.710033295Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.710045814Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"":"downstairs","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.710058773Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.710070502Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"0","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.7100838Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"1","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.710096729Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"client":"2","":"downstairs","session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.710109468Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.710121577Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"d225bb39-e12a-4dc0-8310-ed2f510d09dd is now active with session: c0096fe6-f9b8-4666-ae4d-93214c7516b4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.710136575Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 30,"time":"2025-01-28T17:15:52.710152124Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835,"session_id":"c0096fe6-f9b8-4666-ae4d-93214c7516b4"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:52.710164362Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 453 len 32768 data: 10 9 10 11 10 9 11 12 02/30 Write block 456 len 16384 data: 12 11 10 12 03/30 Write block 320 len 24576 data: 9 10 14 13 15 15 04/30 Read block 138 len 36864 05/30 Write block 493 len 36864 data: 16 16 15 15 12 13 14 13 11 06/30 Write block 528 len 12288 data: 7 11 15 07/30 Read block 73 len 36864 08/30 Read block 227 len 36864 09/30 Flush 10/30 Write block 139 len 8192 data: 18 22 11/30 Write block 129 len 20480 data: 6 6 13 13 12 12/30 Write block 33 len 36864 data: 11 12 10 9 9 11 10 11 10 13/30 Read block 435 len 24576 14/30 Read block 397 len 28672 15/30 Read block 235 len 12288 16/30 Write block 184 len 16384 data: 15 14 11 12 17/30 Read block 305 len 8192 18/30 Read block 218 len 20480 19/30 Read block 469 len 24576 20/30 Read block 20 len 12288 21/30 Write block 198 len 20480 data: 12 12 13 11 13 22/30 Write block 279 len 4096 data: 14 23/30 Read block 200 len 24576 24/30 Read block 566 len 24576 25/30 Read block 238 len 24576 26/30 Write block 451 len 4096 data: 12 27/30 Read block 152 len 24576 28/30 Read block 329 len 32768 29/30 Write block 347 len 12288 data: 16 15 14 30/30 Write block 81 len 40960 data: 13 13 14 14 13 12 13 13 15 15 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 2 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:52.729142165Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1835} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 295 295 295 F F F 1 020-039 160 160 160 294 294 294 T T T 2 040-059 156 156 156 283 283 283 T T T 3 060-079 160 160 160 293 293 293 F F F 4 080-099 160 160 160 293 293 293 T T T 5 100-119 160 160 160 294 294 294 F F F 6 120-139 161 161 161 295 295 295 T T T 7 140-159 161 161 161 295 295 295 T T T 8 160-179 159 159 159 292 292 292 F F F 9 180-199 156 156 156 286 286 286 T T T 10 200-219 158 158 158 289 289 289 T T T 11 220-239 155 155 155 281 281 281 F F F 12 240-259 160 160 160 293 293 293 F F F 13 260-279 159 159 159 292 292 292 T T T 14 280-299 160 160 160 294 294 294 F F F 15 300-319 158 158 158 290 290 290 F F F 16 320-339 162 162 162 296 296 296 F F F 17 340-359 158 158 158 290 290 290 T T T 18 360-379 160 160 160 294 294 294 F F F 19 380-399 159 159 159 292 292 292 F F F 20 400-419 161 161 161 295 295 295 F F F 21 420-439 156 156 156 283 283 283 F F F 22 440-459 162 162 162 296 296 296 T T T 23 460-479 162 162 162 296 296 296 F F F 24 480-499 162 162 162 296 296 296 F F F 25 500-519 162 162 162 296 296 296 F F F 26 520-539 162 162 162 296 296 296 F F F 27 540-559 160 160 160 294 294 294 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 162, Max flush: 296 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 295 295 295 F F F 1 020-039 163 163 163 297 297 297 F F F 2 040-059 163 163 163 297 297 297 F F F 3 060-079 160 160 160 293 293 293 F F F 4 080-099 163 163 163 297 297 297 F F F 5 100-119 160 160 160 294 294 294 F F F 6 120-139 163 163 163 297 297 297 F F F 7 140-159 163 163 163 297 297 297 F F F 8 160-179 159 159 159 292 292 292 F F F 9 180-199 163 163 163 297 297 297 F F F 10 200-219 163 163 163 297 297 297 F F F 11 220-239 155 155 155 281 281 281 F F F 12 240-259 160 160 160 293 293 293 F F F 13 260-279 163 163 163 297 297 297 F F F 14 280-299 160 160 160 294 294 294 F F F 15 300-319 158 158 158 290 290 290 F F F 16 320-339 162 162 162 296 296 296 F F F 17 340-359 163 163 163 297 297 297 F F F 18 360-379 160 160 160 294 294 294 F F F 19 380-399 159 159 159 292 292 292 F F F 20 400-419 161 161 161 295 295 295 F F F 21 420-439 156 156 156 283 283 283 F F F 22 440-459 163 163 163 297 297 297 F F F 23 460-479 162 162 162 296 296 296 F F F 24 480-499 162 162 162 296 296 296 F F F 25 500-519 162 162 162 296 296 296 F F F 26 520-539 162 162 162 296 296 296 F F F 27 540-559 160 160 160 294 294 294 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 163, Max flush: 297 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:56.991348332Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:57.033176859Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.033570161Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.064250893Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.064858104Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.06489972Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.064923128Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.064941106Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843} {"msg":"Crucible 6c51ec29-6fe9-4d7c-afe4-b564b730d3f6 has session id: 9201300a-592a-4532-a64f-c252f1918099","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.06551455Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Upstairs opts: Upstairs UUID: 6c51ec29-6fe9-4d7c-afe4-b564b730d3f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.065563936Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Crucible stats registered with UUID: 6c51ec29-6fe9-4d7c-afe4-b564b730d3f6","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.065588553Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:57.065602562Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066060547Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066085035Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843} {"msg":"6c51ec29-6fe9-4d7c-afe4-b564b730d3f6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066145719Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066163017Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"0","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066188785Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"0","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066203203Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"1","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066217592Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"1","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066231551Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"2","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066245499Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"2","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066258378Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066271797Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066285186Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"ds_connection connected from Ok(127.0.0.1:50669)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066298954Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"ds_connection connected from Ok(127.0.0.1:56544)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066523732Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"ds_connection connected from Ok(127.0.0.1:58548)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066614043Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.066952001Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"2","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.067072339Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"1","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.067254111Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"0","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.067645323Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"1","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.067697058Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"2","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.067993819Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"0","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068217037Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068312348Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"[0]R flush_numbers[0..12]: [295, 297, 297, 293, 297, 294, 297, 297, 292, 297, 297, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068509119Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"[0]R generation[0..12]: [161, 163, 163, 160, 163, 160, 163, 163, 159, 163, 163, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068546445Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068572993Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"[1]R flush_numbers[0..12]: [295, 297, 297, 293, 297, 294, 297, 297, 292, 297, 297, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.06859912Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"[1]R generation[0..12]: [161, 163, 163, 160, 163, 160, 163, 163, 159, 163, 163, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068625068Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068649315Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"[2]R flush_numbers[0..12]: [295, 297, 297, 293, 297, 294, 297, 297, 292, 297, 297, 281]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068676713Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R generation[0..12]: [161, 163, 163, 160, 163, 160, 163, 163, 159, 163, 163, 155]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068710289Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068740236Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068772303Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.06879967Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068826088Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068852185Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"":"downstairs","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068877773Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.06890517Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"0","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068931688Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"1","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068950736Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"client":"2","":"downstairs","session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068964245Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068976463Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"6c51ec29-6fe9-4d7c-afe4-b564b730d3f6 is now active with session: 9201300a-592a-4532-a64f-c252f1918099","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.068989162Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.069004871Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843,"session_id":"9201300a-592a-4532-a64f-c252f1918099"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:15:57.069025639Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 317 len 24576 data: 12 10 7 10 11 15 02/30 Write block 441 len 36864 data: 13 14 16 16 18 17 17 14 11 03/30 Flush 04/30 Write block 429 len 40960 data: 13 15 15 13 16 20 17 15 16 17 05/30 Write block 489 len 28672 data: 12 11 10 14 17 17 16 06/30 Read block 262 len 12288 07/30 Read block 437 len 12288 08/30 Read block 401 len 32768 09/30 Read block 465 len 28672 10/30 Read block 554 len 24576 11/30 Write block 449 len 32768 data: 12 15 13 9 11 10 11 13 12/30 Write block 513 len 28672 data: 16 14 14 19 18 18 19 13/30 Flush 14/30 Write block 32 len 28672 data: 11 12 13 11 10 10 12 15/30 Write block 219 len 32768 data: 10 11 12 16 17 18 17 16 16/30 Read block 305 len 28672 17/30 Read block 85 len 8192 18/30 Read block 71 len 40960 19/30 Write block 213 len 16384 data: 13 13 13 13 20/30 Write block 466 len 20480 data: 12 11 9 10 11 21/30 Read block 428 len 36864 22/30 Read block 164 len 16384 23/30 Write block 550 len 36864 data: 13 13 14 14 16 17 15 15 16 24/30 Read block 449 len 40960 25/30 Write block 466 len 24576 data: 13 12 10 11 12 11 26/30 Read block 105 len 20480 27/30 Write block 225 len 12288 data: 18 17 12 28/30 Write block 88 len 20480 data: 14 16 16 15 13 29/30 Write block 449 len 32768 data: 13 16 14 10 12 11 12 14 30/30 Write block 213 len 24576 data: 14 14 14 14 10 9 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 7 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:15:57.088705852Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1843} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 295 295 295 F F F 1 020-039 163 163 163 297 297 297 T T T 2 040-059 163 163 163 297 297 297 F F F 3 060-079 160 160 160 293 293 293 F F F 4 080-099 163 163 163 297 297 297 T T T 5 100-119 160 160 160 294 294 294 F F F 6 120-139 163 163 163 297 297 297 F F F 7 140-159 163 163 163 297 297 297 F F F 8 160-179 159 159 159 292 292 292 F F F 9 180-199 163 163 163 297 297 297 F F F 10 200-219 163 163 163 297 297 297 T T T 11 220-239 155 155 155 281 281 281 T T T 12 240-259 160 160 160 293 293 293 F F F 13 260-279 163 163 163 297 297 297 F F F 14 280-299 160 160 160 294 294 294 F F F 15 300-319 164 164 164 298 298 298 F F F 16 320-339 164 164 164 298 298 298 F F F 17 340-359 163 163 163 297 297 297 F F F 18 360-379 160 160 160 294 294 294 F F F 19 380-399 159 159 159 292 292 292 F F F 20 400-419 161 161 161 295 295 295 F F F 21 420-439 164 164 164 299 299 299 F F F 22 440-459 164 164 164 299 299 299 T T T 23 460-479 162 162 162 296 296 296 T T T 24 480-499 164 164 164 299 299 299 F F F 25 500-519 164 164 164 299 299 299 F F F 26 520-539 162 162 162 296 296 296 F F F 27 540-559 160 160 160 294 294 294 T T T 28 560-579 142 142 142 257 257 257 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 164, Max flush: 299 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 295 295 295 F F F 1 020-039 165 165 165 300 300 300 F F F 2 040-059 163 163 163 297 297 297 F F F 3 060-079 160 160 160 293 293 293 F F F 4 080-099 165 165 165 300 300 300 F F F 5 100-119 160 160 160 294 294 294 F F F 6 120-139 163 163 163 297 297 297 F F F 7 140-159 163 163 163 297 297 297 F F F 8 160-179 159 159 159 292 292 292 F F F 9 180-199 163 163 163 297 297 297 F F F 10 200-219 165 165 165 300 300 300 F F F 11 220-239 165 165 165 300 300 300 F F F 12 240-259 160 160 160 293 293 293 F F F 13 260-279 163 163 163 297 297 297 F F F 14 280-299 160 160 160 294 294 294 F F F 15 300-319 164 164 164 298 298 298 F F F 16 320-339 164 164 164 298 298 298 F F F 17 340-359 163 163 163 297 297 297 F F F 18 360-379 160 160 160 294 294 294 F F F 19 380-399 159 159 159 292 292 292 F F F 20 400-419 161 161 161 295 295 295 F F F 21 420-439 164 164 164 299 299 299 F F F 22 440-459 165 165 165 300 300 300 F F F 23 460-479 165 165 165 300 300 300 F F F 24 480-499 164 164 164 299 299 299 F F F 25 500-519 164 164 164 299 299 299 F F F 26 520-539 162 162 162 296 296 296 F F F 27 540-559 165 165 165 300 300 300 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 165, Max flush: 300 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.367317252Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.410796089Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.411511979Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.411593911Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.411616369Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.411635087Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851} {"msg":"Crucible a807548a-29b1-48d4-81c8-d5e1663dede8 has session id: 785b287b-284c-458c-b99b-e63504fd445a","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.412265716Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"Upstairs opts: Upstairs UUID: a807548a-29b1-48d4-81c8-d5e1663dede8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.412308382Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"Crucible stats registered with UUID: a807548a-29b1-48d4-81c8-d5e1663dede8","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.4123261Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:01.412339319Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.412764657Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.412788585Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851} {"msg":"a807548a-29b1-48d4-81c8-d5e1663dede8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.412868297Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.412888405Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"0","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.412910453Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"0","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.412924431Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"1","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.41293961Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"1","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.412954728Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"2","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.412968947Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"2","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.412982016Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.413000114Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.413046979Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63176)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.413077496Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50233)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.413234761Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64572)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.41334471Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.413918275Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"0","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.413959971Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"2","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.413990268Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"1","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.414570991Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"1","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.414614967Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"2","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.414691079Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"0","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415204809Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415230617Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"[0]R flush_numbers[0..12]: [295, 300, 297, 293, 300, 294, 297, 297, 292, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415248835Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"[0]R generation[0..12]: [161, 165, 163, 160, 165, 160, 163, 163, 159, 163, 165, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415263504Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415281622Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"[1]R flush_numbers[0..12]: [295, 300, 297, 293, 300, 294, 297, 297, 292, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.41529612Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"[1]R generation[0..12]: [161, 165, 163, 160, 165, 160, 163, 163, 159, 163, 165, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415309839Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415323618Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"[2]R flush_numbers[0..12]: [295, 300, 297, 293, 300, 294, 297, 297, 292, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415337196Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"[2]R generation[0..12]: [161, 165, 163, 160, 165, 160, 163, 163, 159, 163, 165, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415351475Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415365514Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415378163Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415390991Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.41540346Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415415999Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"":"downstairs","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415428798Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"session_id":"785b287b-284c-458c-b99b-e63504fd445a"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415454215Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"0","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415480343Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"1","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.41551085Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"client":"2","":"downstairs","session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415538527Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415563245Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"a807548a-29b1-48d4-81c8-d5e1663dede8 is now active with session: 785b287b-284c-458c-b99b-e63504fd445a","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415588832Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.41561388Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851,"session_id":"785b287b-284c-458c-b99b-e63504fd445a"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:01.415638727Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 60 len 32768 data: 8 12 12 12 12 14 15 15 03/30 Write block 427 len 16384 data: 11 13 14 16 04/30 Write block 147 len 4096 data: 9 05/30 Write block 387 len 12288 data: 12 12 11 06/30 Write block 63 len 32768 data: 13 13 15 16 16 14 15 11 07/30 Write block 44 len 28672 data: 11 9 9 11 9 10 10 08/30 Write block 498 len 28672 data: 14 15 14 12 12 16 16 09/30 Read block 507 len 32768 10/30 Write block 242 len 36864 data: 20 16 16 15 13 15 15 11 13 11/30 Flush 12/30 Flush 13/30 Flush 14/30 Write block 558 len 8192 data: 17 13 15/30 Read block 347 len 36864 16/30 Write block 164 len 16384 data: 22 17 14 14 17/30 Read block 482 len 12288 18/30 Read block 364 len 28672 19/30 Read block 176 len 36864 20/30 Read block 457 len 32768 21/30 Write block 367 len 20480 data: 8 9 9 9 10 22/30 Write block 444 len 20480 data: 17 19 18 18 15 23/30 Read block 448 len 4096 24/30 Read block 230 len 8192 25/30 Read block 555 len 8192 26/30 Write block 19 len 8192 data: 12 11 27/30 Write block 9 len 12288 data: 14 14 14 28/30 Write block 130 len 24576 data: 7 14 14 13 11 11 29/30 Write block 397 len 20480 data: 10 12 14 15 16 30/30 Write block 175 len 8192 data: 18 17 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 2 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:01.432178347Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1851} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 295 295 295 T T T 1 020-039 165 165 165 300 300 300 T T T 2 040-059 166 166 166 302 302 302 F F F 3 060-079 166 166 166 302 302 302 F F F 4 080-099 165 165 165 300 300 300 F F F 5 100-119 160 160 160 294 294 294 F F F 6 120-139 163 163 163 297 297 297 T T T 7 140-159 166 166 166 302 302 302 F F F 8 160-179 159 159 159 292 292 292 T T T 9 180-199 163 163 163 297 297 297 F F F 10 200-219 165 165 165 300 300 300 F F F 11 220-239 165 165 165 300 300 300 F F F 12 240-259 166 166 166 302 302 302 F F F 13 260-279 163 163 163 297 297 297 F F F 14 280-299 160 160 160 294 294 294 F F F 15 300-319 164 164 164 298 298 298 F F F 16 320-339 164 164 164 298 298 298 F F F 17 340-359 163 163 163 297 297 297 F F F 18 360-379 160 160 160 294 294 294 T T T 19 380-399 166 166 166 302 302 302 T T T 20 400-419 161 161 161 295 295 295 T T T 21 420-439 166 166 166 302 302 302 F F F 22 440-459 165 165 165 300 300 300 T T T 23 460-479 165 165 165 300 300 300 F F F 24 480-499 166 166 166 302 302 302 F F F 25 500-519 166 166 166 302 302 302 F F F 26 520-539 162 162 162 296 296 296 F F F 27 540-559 165 165 165 300 300 300 T T T 28 560-579 142 142 142 257 257 257 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 166, Max flush: 302 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 303 303 303 F F F 1 020-039 167 167 167 303 303 303 F F F 2 040-059 166 166 166 302 302 302 F F F 3 060-079 166 166 166 302 302 302 F F F 4 080-099 165 165 165 300 300 300 F F F 5 100-119 160 160 160 294 294 294 F F F 6 120-139 167 167 167 303 303 303 F F F 7 140-159 166 166 166 302 302 302 F F F 8 160-179 167 167 167 303 303 303 F F F 9 180-199 163 163 163 297 297 297 F F F 10 200-219 165 165 165 300 300 300 F F F 11 220-239 165 165 165 300 300 300 F F F 12 240-259 166 166 166 302 302 302 F F F 13 260-279 163 163 163 297 297 297 F F F 14 280-299 160 160 160 294 294 294 F F F 15 300-319 164 164 164 298 298 298 F F F 16 320-339 164 164 164 298 298 298 F F F 17 340-359 163 163 163 297 297 297 F F F 18 360-379 167 167 167 303 303 303 F F F 19 380-399 167 167 167 303 303 303 F F F 20 400-419 167 167 167 303 303 303 F F F 21 420-439 166 166 166 302 302 302 F F F 22 440-459 167 167 167 303 303 303 F F F 23 460-479 165 165 165 300 300 300 F F F 24 480-499 166 166 166 302 302 302 F F F 25 500-519 166 166 166 302 302 302 F F F 26 520-539 162 162 162 296 296 296 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 167, Max flush: 303 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.68989595Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.732886154Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.733504384Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.7335444Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.733558569Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.733573107Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859} {"msg":"Crucible 6efb8743-1958-4aaf-a526-af3e8b9676c4 has session id: 49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.73416137Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Upstairs opts: Upstairs UUID: 6efb8743-1958-4aaf-a526-af3e8b9676c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734198286Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Crucible stats registered with UUID: 6efb8743-1958-4aaf-a526-af3e8b9676c4","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734213505Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:05.734226093Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734630934Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734654792Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859} {"msg":"6efb8743-1958-4aaf-a526-af3e8b9676c4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734722195Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734743783Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"0","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734765981Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"0","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.73477969Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"1","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734793808Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"1","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734806547Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"2","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734820046Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"2","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734832794Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734846253Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734867301Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:41707)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.734963182Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:60116)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.735001478Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:05.735126186Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:05.735168592Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:05.735200878Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.735247624Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.73570185Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"2","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.735730067Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"0","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.736301521Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"2","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.736457196Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"0","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.736878145Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:05.736960367Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.736296222Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:40309)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.736455566Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.737479147Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"1","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738035423Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"1","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"[0]R flush_numbers[0..12]: [303, 303, 302, 302, 300, 294, 303, 302, 303, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738648143Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"[0]R generation[0..12]: [167, 167, 166, 166, 165, 160, 167, 166, 167, 163, 165, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738672561Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738687579Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"[1]R flush_numbers[0..12]: [303, 303, 302, 302, 300, 294, 303, 302, 303, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738701388Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"[1]R generation[0..12]: [167, 167, 166, 166, 165, 160, 167, 166, 167, 163, 165, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738720316Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738734455Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"[2]R flush_numbers[0..12]: [303, 303, 302, 302, 300, 294, 303, 302, 303, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738748083Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"[2]R generation[0..12]: [167, 167, 166, 166, 165, 160, 167, 166, 167, 163, 165, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738761622Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738775031Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-01-28T17:16:15.73878777Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738807477Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738819996Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738832475Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"":"downstairs","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738845174Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738856693Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"0","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738869502Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"1","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.73888206Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"client":"2","":"downstairs","session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738894859Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738914457Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"6efb8743-1958-4aaf-a526-af3e8b9676c4 is now active with session: 49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738927046Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738938675Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859,"session_id":"49e4a198-f3e0-47b4-97eb-a54a6fe1b9fb"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:15.738950464Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 277 len 16384 data: 12 15 15 10 02/30 Read block 487 len 32768 03/30 Read block 255 len 32768 04/30 Write block 49 len 20480 data: 11 11 13 14 12 05/30 Read block 444 len 28672 06/30 Read block 552 len 4096 07/30 Read block 78 len 20480 08/30 Write block 438 len 28672 data: 18 15 14 14 15 17 18 09/30 Read block 536 len 12288 10/30 Write block 387 len 20480 data: 13 13 12 10 13 11/30 Read block 495 len 24576 12/30 Read block 551 len 8192 13/30 Write block 498 len 28672 data: 15 16 15 13 13 17 17 14/30 Read block 279 len 20480 15/30 Write block 280 len 28672 data: 11 9 9 8 6 9 8 16/30 Read block 413 len 36864 17/30 Write block 571 len 32768 data: 15 15 15 13 10 10 16 18 18/30 Flush 19/30 Write block 88 len 4096 data: 15 20/30 Write block 369 len 24576 data: 10 10 11 10 7 8 21/30 Read block 495 len 8192 22/30 Write block 448 len 28672 data: 16 14 17 15 11 13 12 23/30 Write block 460 len 4096 data: 13 24/30 Read block 10 len 16384 25/30 Write block 534 len 16384 data: 12 14 11 15 26/30 Read block 40 len 8192 27/30 Read block 306 len 32768 28/30 Write block 442 len 4096 data: 16 29/30 Read block 570 len 28672 30/30 Write block 267 len 4096 data: 17 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 1 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:15.759271375Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1859} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 303 303 303 F F F 1 020-039 167 167 167 303 303 303 F F F 2 040-059 168 168 168 304 304 304 F F F 3 060-079 166 166 166 302 302 302 F F F 4 080-099 165 165 165 300 300 300 T T T 5 100-119 160 160 160 294 294 294 F F F 6 120-139 167 167 167 303 303 303 F F F 7 140-159 166 166 166 302 302 302 F F F 8 160-179 167 167 167 303 303 303 F F F 9 180-199 163 163 163 297 297 297 F F F 10 200-219 165 165 165 300 300 300 F F F 11 220-239 165 165 165 300 300 300 F F F 12 240-259 166 166 166 302 302 302 F F F 13 260-279 168 168 168 304 304 304 T T T 14 280-299 168 168 168 304 304 304 F F F 15 300-319 164 164 164 298 298 298 F F F 16 320-339 164 164 164 298 298 298 F F F 17 340-359 163 163 163 297 297 297 F F F 18 360-379 167 167 167 303 303 303 T T T 19 380-399 168 168 168 304 304 304 F F F 20 400-419 167 167 167 303 303 303 F F F 21 420-439 168 168 168 304 304 304 F F F 22 440-459 168 168 168 304 304 304 T T T 23 460-479 165 165 165 300 300 300 T T T 24 480-499 168 168 168 304 304 304 F F F 25 500-519 168 168 168 304 304 304 F F F 26 520-539 162 162 162 296 296 296 T T T 27 540-559 167 167 167 303 303 303 F F F 28 560-579 168 168 168 304 304 304 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 168, Max flush: 304 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 303 303 303 F F F 1 020-039 167 167 167 303 303 303 F F F 2 040-059 168 168 168 304 304 304 F F F 3 060-079 166 166 166 302 302 302 F F F 4 080-099 169 169 169 305 305 305 F F F 5 100-119 160 160 160 294 294 294 F F F 6 120-139 167 167 167 303 303 303 F F F 7 140-159 166 166 166 302 302 302 F F F 8 160-179 167 167 167 303 303 303 F F F 9 180-199 163 163 163 297 297 297 F F F 10 200-219 165 165 165 300 300 300 F F F 11 220-239 165 165 165 300 300 300 F F F 12 240-259 166 166 166 302 302 302 F F F 13 260-279 169 169 169 305 305 305 F F F 14 280-299 168 168 168 304 304 304 F F F 15 300-319 164 164 164 298 298 298 F F F 16 320-339 164 164 164 298 298 298 F F F 17 340-359 163 163 163 297 297 297 F F F 18 360-379 169 169 169 305 305 305 F F F 19 380-399 168 168 168 304 304 304 F F F 20 400-419 167 167 167 303 303 303 F F F 21 420-439 168 168 168 304 304 304 F F F 22 440-459 169 169 169 305 305 305 F F F 23 460-479 169 169 169 305 305 305 F F F 24 480-499 168 168 168 304 304 304 F F F 25 500-519 168 168 168 304 304 304 F F F 26 520-539 169 169 169 305 305 305 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 168 168 168 304 304 304 F F F 29 580-599 156 156 156 286 286 286 F F F Max gen: 169, Max flush: 305 Begin loop 85 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.003267436Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.047671782Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.048431528Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.048482563Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.048500981Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.04851898Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867} {"msg":"Crucible bc9a3019-1966-4a58-a688-e5c78f5baed9 has session id: ad26f009-2fb8-4739-ac64-cbfec5bc19da","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.04902146Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Upstairs opts: Upstairs UUID: bc9a3019-1966-4a58-a688-e5c78f5baed9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049066886Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Crucible stats registered with UUID: bc9a3019-1966-4a58-a688-e5c78f5baed9","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049090464Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:20.049111772Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049633431Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049662158Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867} {"msg":"bc9a3019-1966-4a58-a688-e5c78f5baed9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049723452Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.04974859Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"0","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049772447Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"0","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049786576Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"1","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049803984Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"1","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049817543Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"2","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049831772Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"2","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.04984488Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049858279Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049871798Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"ds_connection connected from Ok(127.0.0.1:65211)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.049891306Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"ds_connection connected from Ok(127.0.0.1:37007)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.050039641Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"ds_connection connected from Ok(127.0.0.1:45334)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.050175188Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.050645692Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"0","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.050807407Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"1","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.050835554Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"2","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.051189969Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"0","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.051341235Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"2","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.051449894Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"1","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.051829207Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.051950455Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"[0]R flush_numbers[0..12]: [303, 303, 304, 302, 305, 294, 303, 302, 303, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052026308Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"[0]R generation[0..12]: [167, 167, 168, 166, 169, 160, 167, 166, 167, 163, 165, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052048236Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052062924Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"[1]R flush_numbers[0..12]: [303, 303, 304, 302, 305, 294, 303, 302, 303, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052076803Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"[1]R generation[0..12]: [167, 167, 168, 166, 169, 160, 167, 166, 167, 163, 165, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052090702Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.05210463Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"[2]R flush_numbers[0..12]: [303, 303, 304, 302, 305, 294, 303, 302, 303, 297, 300, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052118589Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"[2]R generation[0..12]: [167, 167, 168, 166, 169, 160, 167, 166, 167, 163, 165, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052132238Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052145966Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052158665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052177463Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052194382Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.05220753Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"":"downstairs","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052220529Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052232168Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"0","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052245237Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"1","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052257985Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"client":"2","":"downstairs","session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052270724Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052282783Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"bc9a3019-1966-4a58-a688-e5c78f5baed9 is now active with session: ad26f009-2fb8-4739-ac64-cbfec5bc19da","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052295292Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052307001Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867,"session_id":"ad26f009-2fb8-4739-ac64-cbfec5bc19da"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:20.052318939Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 135 len 4096 data: 12 02/30 Write block 270 len 32768 data: 12 13 10 10 13 13 13 13 03/30 Read block 115 len 16384 04/30 Write block 478 len 4096 data: 10 05/30 Write block 329 len 12288 data: 15 14 10 06/30 Flush 07/30 Write block 102 len 12288 data: 9 7 10 08/30 Read block 25 len 4096 09/30 Write block 513 len 4096 data: 17 10/30 Read block 32 len 8192 11/30 Read block 387 len 4096 12/30 Read block 123 len 32768 13/30 Write block 38 len 4096 data: 13 14/30 Read block 230 len 32768 15/30 Read block 514 len 40960 16/30 Flush 17/30 Write block 593 len 16384 data: 17 14 10 10 18/30 Read block 414 len 12288 19/30 Read block 335 len 36864 20/30 Write block 42 len 12288 data: 10 11 12 21/30 Read block 571 len 16384 22/30 Write block 429 len 36864 data: 15 17 16 14 17 21 18 16 17 23/30 Read block 458 len 20480 24/30 Write block 248 len 20480 data: 16 12 14 12 14 25/30 Read block 472 len 40960 26/30 Write block 461 len 28672 data: 13 15 14 15 16 14 13 27/30 Write block 252 len 20480 data: 15 14 14 11 13 28/30 Read block 341 len 12288 29/30 Write block 193 len 36864 data: 13 13 14 14 13 13 13 14 12 30/30 Read block 560 len 40960 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 4 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 13 14 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:20.071175453Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1867} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 303 303 303 F F F 1 020-039 170 170 170 307 307 307 F F F 2 040-059 168 168 168 304 304 304 T T T 3 060-079 166 166 166 302 302 302 F F F 4 080-099 169 169 169 305 305 305 F F F 5 100-119 170 170 170 307 307 307 F F F 6 120-139 170 170 170 306 306 306 F F F 7 140-159 166 166 166 302 302 302 F F F 8 160-179 167 167 167 303 303 303 F F F 9 180-199 163 163 163 297 297 297 T T T 10 200-219 165 165 165 300 300 300 T T T 11 220-239 165 165 165 300 300 300 F F F 12 240-259 166 166 166 302 302 302 T T T 13 260-279 170 170 170 306 306 306 F F F 14 280-299 168 168 168 304 304 304 F F F 15 300-319 164 164 164 298 298 298 F F F 16 320-339 170 170 170 306 306 306 F F F 17 340-359 163 163 163 297 297 297 F F F 18 360-379 169 169 169 305 305 305 F F F 19 380-399 168 168 168 304 304 304 F F F 20 400-419 167 167 167 303 303 303 F F F 21 420-439 168 168 168 304 304 304 T T T 22 440-459 169 169 169 305 305 305 F F F 23 460-479 170 170 170 306 306 306 T T T 24 480-499 168 168 168 304 304 304 F F F 25 500-519 170 170 170 307 307 307 F F F 26 520-539 169 169 169 305 305 305 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 168 168 168 304 304 304 F F F 29 580-599 156 156 156 286 286 286 T T T Max gen: 170, Max flush: 307 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 303 303 303 F F F 1 020-039 170 170 170 307 307 307 F F F 2 040-059 171 171 171 308 308 308 F F F 3 060-079 166 166 166 302 302 302 F F F 4 080-099 169 169 169 305 305 305 F F F 5 100-119 170 170 170 307 307 307 F F F 6 120-139 170 170 170 306 306 306 F F F 7 140-159 166 166 166 302 302 302 F F F 8 160-179 167 167 167 303 303 303 F F F 9 180-199 171 171 171 308 308 308 F F F 10 200-219 171 171 171 308 308 308 F F F 11 220-239 165 165 165 300 300 300 F F F 12 240-259 171 171 171 308 308 308 F F F 13 260-279 170 170 170 306 306 306 F F F 14 280-299 168 168 168 304 304 304 F F F 15 300-319 164 164 164 298 298 298 F F F 16 320-339 170 170 170 306 306 306 F F F 17 340-359 163 163 163 297 297 297 F F F 18 360-379 169 169 169 305 305 305 F F F 19 380-399 168 168 168 304 304 304 F F F 20 400-419 167 167 167 303 303 303 F F F 21 420-439 171 171 171 308 308 308 F F F 22 440-459 169 169 169 305 305 305 F F F 23 460-479 171 171 171 308 308 308 F F F 24 480-499 168 168 168 304 304 304 F F F 25 500-519 170 170 170 307 307 307 F F F 26 520-539 169 169 169 305 305 305 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 168 168 168 304 304 304 F F F 29 580-599 171 171 171 308 308 308 F F F Max gen: 171, Max flush: 308 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.313200677Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.35619902Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.356819559Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.356870044Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.356888113Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.356906661Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875} {"msg":"Crucible b5fb7297-811f-4ca2-8b5a-f9a52ccbe88e has session id: d5cecb81-36bf-4683-b81c-4992a8e7c464","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.357407012Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Upstairs opts: Upstairs UUID: b5fb7297-811f-4ca2-8b5a-f9a52ccbe88e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.357465896Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Crucible stats registered with UUID: b5fb7297-811f-4ca2-8b5a-f9a52ccbe88e","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.357495624Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:24.357525021Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.357854258Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.357884946Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875} {"msg":"b5fb7297-811f-4ca2-8b5a-f9a52ccbe88e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.357917972Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.35793975Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"0","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.357976397Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"0","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.358005464Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"1","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.358033771Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"1","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.358061788Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"2","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.358090395Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"2","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.358129352Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.358156769Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.358181487Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"ds_connection connected from Ok(127.0.0.1:43511)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.358205454Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"ds_connection connected from Ok(127.0.0.1:62277)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.358314974Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"ds_connection connected from Ok(127.0.0.1:60387)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.358340741Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.358700746Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"2","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.358828224Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"0","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.359198028Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"2","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.359327235Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"0","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.359817087Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.359843005Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.360777644Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"1","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.36132602Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"1","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"[0]R flush_numbers[0..12]: [303, 307, 308, 302, 305, 307, 306, 302, 303, 308, 308, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.361816073Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"[0]R generation[0..12]: [167, 170, 171, 166, 169, 170, 170, 166, 167, 171, 171, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.36184109Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.361856229Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"[1]R flush_numbers[0..12]: [303, 307, 308, 302, 305, 307, 306, 302, 303, 308, 308, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.361870187Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"[1]R generation[0..12]: [167, 170, 171, 166, 169, 170, 170, 166, 167, 171, 171, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.361883716Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.361897805Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"[2]R flush_numbers[0..12]: [303, 307, 308, 302, 305, 307, 306, 302, 303, 308, 308, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.361916283Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"[2]R generation[0..12]: [167, 170, 171, 166, 169, 170, 170, 166, 167, 171, 171, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.361932381Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.361954509Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.361970278Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.361986336Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.362001895Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.362017483Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"":"downstairs","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.362033651Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.36204822Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"0","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.362064409Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"1","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.362080307Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"client":"2","":"downstairs","session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.362096296Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.362128232Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"b5fb7297-811f-4ca2-8b5a-f9a52ccbe88e is now active with session: d5cecb81-36bf-4683-b81c-4992a8e7c464","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.362144491Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.362161609Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875,"session_id":"d5cecb81-36bf-4683-b81c-4992a8e7c464"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:24.362176878Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 401 len 12288 data: 17 14 18 02/30 Read block 371 len 8192 03/30 Read block 174 len 8192 04/30 Write block 531 len 32768 data: 18 16 17 13 15 12 16 15 05/30 Write block 402 len 8192 data: 15 19 06/30 Write block 109 len 12288 data: 12 12 10 07/30 Flush 08/30 Write block 104 len 28672 data: 11 10 10 13 13 13 13 09/30 Read block 246 len 20480 10/30 Read block 314 len 20480 11/30 Read block 73 len 40960 12/30 Read block 255 len 4096 13/30 Read block 143 len 16384 14/30 Flush 15/30 Write block 138 len 24576 data: 17 19 23 19 15 13 16/30 Write block 133 len 40960 data: 14 12 13 11 15 18 20 24 20 16 17/30 Read block 131 len 20480 18/30 Read block 418 len 40960 19/30 Write block 158 len 28672 data: 16 14 15 15 16 18 23 20/30 Read block 300 len 36864 21/30 Flush 22/30 Write block 330 len 36864 data: 15 11 9 12 14 12 12 14 12 23/30 Read block 365 len 16384 24/30 Write block 196 len 32768 data: 15 14 14 14 15 13 14 15 25/30 Write block 10 len 8192 data: 15 15 26/30 Write block 18 len 20480 data: 10 13 12 9 12 27/30 Write block 198 len 16384 data: 15 15 16 14 28/30 Write block 402 len 32768 data: 16 20 16 16 14 12 16 14 29/30 Write block 316 len 36864 data: 14 13 11 8 11 12 16 14 16 30/30 Write block 496 len 32768 data: 16 13 16 17 16 14 14 18 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:24.38073895Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1875} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 303 303 303 T T T 1 020-039 170 170 170 307 307 307 T T T 2 040-059 171 171 171 308 308 308 F F F 3 060-079 166 166 166 302 302 302 F F F 4 080-099 169 169 169 305 305 305 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 172 172 172 311 311 311 F F F 7 140-159 172 172 172 311 311 311 F F F 8 160-179 172 172 172 311 311 311 F F F 9 180-199 171 171 171 308 308 308 T T T 10 200-219 171 171 171 308 308 308 T T T 11 220-239 165 165 165 300 300 300 F F F 12 240-259 171 171 171 308 308 308 F F F 13 260-279 170 170 170 306 306 306 F F F 14 280-299 168 168 168 304 304 304 F F F 15 300-319 164 164 164 298 298 298 T T T 16 320-339 170 170 170 306 306 306 T T T 17 340-359 163 163 163 297 297 297 F F F 18 360-379 169 169 169 305 305 305 F F F 19 380-399 168 168 168 304 304 304 F F F 20 400-419 172 172 172 309 309 309 T T T 21 420-439 171 171 171 308 308 308 F F F 22 440-459 169 169 169 305 305 305 F F F 23 460-479 171 171 171 308 308 308 F F F 24 480-499 168 168 168 304 304 304 T T T 25 500-519 170 170 170 307 307 307 T T T 26 520-539 172 172 172 309 309 309 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 168 168 168 304 304 304 F F F 29 580-599 171 171 171 308 308 308 F F F Max gen: 172, Max flush: 311 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 312 312 312 F F F 1 020-039 173 173 173 312 312 312 F F F 2 040-059 171 171 171 308 308 308 F F F 3 060-079 166 166 166 302 302 302 F F F 4 080-099 169 169 169 305 305 305 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 172 172 172 311 311 311 F F F 7 140-159 172 172 172 311 311 311 F F F 8 160-179 172 172 172 311 311 311 F F F 9 180-199 173 173 173 312 312 312 F F F 10 200-219 173 173 173 312 312 312 F F F 11 220-239 165 165 165 300 300 300 F F F 12 240-259 171 171 171 308 308 308 F F F 13 260-279 170 170 170 306 306 306 F F F 14 280-299 168 168 168 304 304 304 F F F 15 300-319 173 173 173 312 312 312 F F F 16 320-339 173 173 173 312 312 312 F F F 17 340-359 163 163 163 297 297 297 F F F 18 360-379 169 169 169 305 305 305 F F F 19 380-399 168 168 168 304 304 304 F F F 20 400-419 173 173 173 312 312 312 F F F 21 420-439 171 171 171 308 308 308 F F F 22 440-459 169 169 169 305 305 305 F F F 23 460-479 171 171 171 308 308 308 F F F 24 480-499 173 173 173 312 312 312 F F F 25 500-519 173 173 173 312 312 312 F F F 26 520-539 172 172 172 309 309 309 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 168 168 168 304 304 304 F F F 29 580-599 171 171 171 308 308 308 F F F Max gen: 173, Max flush: 312 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.635760926Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:28.678293155Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.678667668Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.710269601Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.710838046Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.710884202Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.71090131Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.710923978Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883} {"msg":"Crucible f9ad55c7-7610-454b-8985-1427faf66c93 has session id: e23c3c6a-cad3-49fd-8f1f-fccf841d3da8","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.711526459Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Upstairs opts: Upstairs UUID: f9ad55c7-7610-454b-8985-1427faf66c93, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.711578024Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Crucible stats registered with UUID: f9ad55c7-7610-454b-8985-1427faf66c93","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.711598332Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:28.711611281Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.711980435Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712003262Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883} {"msg":"f9ad55c7-7610-454b-8985-1427faf66c93 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712050258Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712076566Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"0","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712099743Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"0","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712113872Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"1","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.71212783Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"1","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712145659Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"2","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712160957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"2","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712185235Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712211282Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.71223634Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"ds_connection connected from Ok(127.0.0.1:65143)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712265337Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"ds_connection connected from Ok(127.0.0.1:54690)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712292474Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"ds_connection connected from Ok(127.0.0.1:63415)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712326631Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712900455Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"1","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.712934792Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"2","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.7130565Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"0","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.713527354Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"2","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.713664421Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"1","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.713744363Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"0","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714077561Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714243104Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"[0]R flush_numbers[0..12]: [312, 312, 308, 302, 305, 310, 311, 311, 311, 312, 312, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714378961Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"[0]R generation[0..12]: [173, 173, 171, 166, 169, 172, 172, 172, 172, 173, 173, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714400299Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714415168Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"[1]R flush_numbers[0..12]: [312, 312, 308, 302, 305, 310, 311, 311, 311, 312, 312, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714428966Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"[1]R generation[0..12]: [173, 173, 171, 166, 169, 172, 172, 172, 172, 173, 173, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714442325Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714455914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"[2]R flush_numbers[0..12]: [312, 312, 308, 302, 305, 310, 311, 311, 311, 312, 312, 300]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714469193Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"[2]R generation[0..12]: [173, 173, 171, 166, 169, 172, 172, 172, 172, 173, 173, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714482401Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.71449592Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714508629Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714527507Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714543695Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714563183Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"":"downstairs","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714581151Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.71459596Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"0","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714612299Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"1","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714628557Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"client":"2","":"downstairs","session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714644665Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714659554Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"f9ad55c7-7610-454b-8985-1427faf66c93 is now active with session: e23c3c6a-cad3-49fd-8f1f-fccf841d3da8","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714675093Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:28.714689631Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883,"session_id":"e23c3c6a-cad3-49fd-8f1f-fccf841d3da8"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-28T17:16:28.71470461Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 467 len 8192 data: 14 11 02/30 Flush 03/30 Write block 410 len 24576 data: 14 10 9 9 10 10 04/30 Write block 2 len 24576 data: 4 5 6 10 9 12 05/30 Read block 159 len 32768 06/30 Write block 335 len 28672 data: 13 13 15 13 13 12 12 07/30 Write block 223 len 28672 data: 18 19 19 18 13 15 15 08/30 Read block 275 len 20480 09/30 Read block 478 len 16384 10/30 Read block 65 len 36864 11/30 Write block 267 len 36864 data: 18 14 12 13 14 11 11 14 14 12/30 Write block 47 len 24576 data: 12 10 12 12 14 15 13/30 Flush 14/30 Flush 15/30 Write block 202 len 28672 data: 15 16 16 15 14 14 14 16/30 Read block 531 len 12288 17/30 Write block 560 len 24576 data: 12 10 11 8 7 7 18/30 Read block 213 len 40960 19/30 Read block 214 len 40960 20/30 Read block 514 len 20480 21/30 Write block 594 len 8192 data: 15 11 22/30 Write block 519 len 28672 data: 20 19 17 13 12 13 11 23/30 Read block 482 len 8192 24/30 Write block 420 len 12288 data: 12 12 11 25/30 Write block 265 len 32768 data: 17 18 19 15 13 14 15 12 26/30 Read block 389 len 32768 27/30 Read block 130 len 8192 28/30 Read block 260 len 16384 29/30 Write block 272 len 40960 data: 13 12 15 15 14 14 16 16 12 10 30/30 Write block 29 len 36864 data: 12 12 12 12 13 14 12 11 11 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 7 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:28.734917062Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1883} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 314 314 314 F F F 1 020-039 173 173 173 312 312 312 T T T 2 040-059 174 174 174 314 314 314 F F F 3 060-079 166 166 166 302 302 302 F F F 4 080-099 169 169 169 305 305 305 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 172 172 172 311 311 311 F F F 7 140-159 172 172 172 311 311 311 F F F 8 160-179 172 172 172 311 311 311 F F F 9 180-199 173 173 173 312 312 312 F F F 10 200-219 173 173 173 312 312 312 T T T 11 220-239 174 174 174 314 314 314 F F F 12 240-259 171 171 171 308 308 308 F F F 13 260-279 174 174 174 314 314 314 T T T 14 280-299 168 168 168 304 304 304 T T T 15 300-319 173 173 173 312 312 312 F F F 16 320-339 174 174 174 314 314 314 F F F 17 340-359 174 174 174 314 314 314 F F F 18 360-379 169 169 169 305 305 305 F F F 19 380-399 168 168 168 304 304 304 F F F 20 400-419 174 174 174 314 314 314 F F F 21 420-439 171 171 171 308 308 308 T T T 22 440-459 169 169 169 305 305 305 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 173 173 173 312 312 312 F F F 25 500-519 173 173 173 312 312 312 T T T 26 520-539 172 172 172 309 309 309 T T T 27 540-559 167 167 167 303 303 303 F F F 28 560-579 168 168 168 304 304 304 T T T 29 580-599 171 171 171 308 308 308 T T T Max gen: 174, Max flush: 314 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 314 314 314 F F F 1 020-039 175 175 175 315 315 315 F F F 2 040-059 174 174 174 314 314 314 F F F 3 060-079 166 166 166 302 302 302 F F F 4 080-099 169 169 169 305 305 305 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 172 172 172 311 311 311 F F F 7 140-159 172 172 172 311 311 311 F F F 8 160-179 172 172 172 311 311 311 F F F 9 180-199 173 173 173 312 312 312 F F F 10 200-219 175 175 175 315 315 315 F F F 11 220-239 174 174 174 314 314 314 F F F 12 240-259 171 171 171 308 308 308 F F F 13 260-279 175 175 175 315 315 315 F F F 14 280-299 175 175 175 315 315 315 F F F 15 300-319 173 173 173 312 312 312 F F F 16 320-339 174 174 174 314 314 314 F F F 17 340-359 174 174 174 314 314 314 F F F 18 360-379 169 169 169 305 305 305 F F F 19 380-399 168 168 168 304 304 304 F F F 20 400-419 174 174 174 314 314 314 F F F 21 420-439 175 175 175 315 315 315 F F F 22 440-459 169 169 169 305 305 305 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 173 173 173 312 312 312 F F F 25 500-519 175 175 175 315 315 315 F F F 26 520-539 175 175 175 315 315 315 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 175 175 175 315 315 315 F F F 29 580-599 175 175 175 315 315 315 F F F Max gen: 175, Max flush: 315 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.026451191Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.070764826Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.071374157Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.071414823Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.071430031Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.07144441Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891} {"msg":"Crucible b0ee68d8-0070-468e-bfe4-074584b75446 has session id: c0d14570-098e-4e82-8b87-de785d374c96","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072016604Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Upstairs opts: Upstairs UUID: b0ee68d8-0070-468e-bfe4-074584b75446, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.07206365Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Crucible stats registered with UUID: b0ee68d8-0070-468e-bfe4-074584b75446","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072087678Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:33.072105926Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.07247187Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072495028Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891} {"msg":"b0ee68d8-0070-468e-bfe4-074584b75446 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072540693Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072567621Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"0","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072591798Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"0","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072606307Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"1","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072624065Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"1","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072647853Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"2","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.07267401Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"2","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072699598Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072726105Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072753623Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"ds_connection connected from Ok(127.0.0.1:52756)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072803648Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"ds_connection connected from Ok(127.0.0.1:63452)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.072834625Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"ds_connection connected from Ok(127.0.0.1:51210)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.073010708Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.073414888Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"0","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.073453965Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"1","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.073556075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"2","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074098452Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"0","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074127539Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"2","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074143617Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"1","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074666406Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074692424Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"[0]R flush_numbers[0..12]: [314, 315, 314, 302, 305, 310, 311, 311, 311, 312, 315, 314]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074710442Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"[0]R generation[0..12]: [174, 175, 174, 166, 169, 172, 172, 172, 172, 173, 175, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074725251Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074739399Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"[1]R flush_numbers[0..12]: [314, 315, 314, 302, 305, 310, 311, 311, 311, 312, 315, 314]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074753098Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"[1]R generation[0..12]: [174, 175, 174, 166, 169, 172, 172, 172, 172, 173, 175, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074766747Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074783995Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"[2]R flush_numbers[0..12]: [314, 315, 314, 302, 305, 310, 311, 311, 311, 312, 315, 314]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074797863Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"[2]R generation[0..12]: [174, 175, 174, 166, 169, 172, 172, 172, 172, 173, 175, 174]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074811332Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074825081Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.0748378Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074850829Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-28T17:16:33.074863497Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074882015Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"":"downstairs","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074894984Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074906883Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"0","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074920002Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"1","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.07493285Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"client":"2","":"downstairs","session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074945789Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074958318Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"b0ee68d8-0070-468e-bfe4-074584b75446 is now active with session: c0d14570-098e-4e82-8b87-de785d374c96","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074970877Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074984645Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891,"session_id":"c0d14570-098e-4e82-8b87-de785d374c96"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:33.074996684Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 297 len 28672 data: 12 13 11 14 16 13 14 02/30 Read block 247 len 28672 03/30 Read block 339 len 32768 04/30 Write block 191 len 8192 data: 13 14 05/30 Flush 06/30 Write block 310 len 12288 data: 20 16 13 07/30 Read block 294 len 40960 08/30 Write block 202 len 40960 data: 16 17 17 16 15 15 15 15 17 15 09/30 Read block 328 len 28672 10/30 Write block 257 len 4096 data: 11 11/30 Read block 134 len 8192 12/30 Write block 587 len 24576 data: 14 17 21 18 18 18 13/30 Write block 577 len 40960 data: 17 19 14 14 15 20 20 21 23 14 14/30 Read block 185 len 4096 15/30 Read block 269 len 32768 16/30 Read block 36 len 8192 17/30 Read block 389 len 28672 18/30 Read block 264 len 40960 19/30 Write block 227 len 32768 data: 14 16 16 12 10 8 8 11 20/30 Write block 69 len 4096 data: 16 21/30 Flush 22/30 Flush 23/30 Write block 447 len 8192 data: 19 17 24/30 Read block 342 len 12288 25/30 Read block 94 len 40960 26/30 Write block 18 len 36864 data: 11 14 13 10 13 12 12 13 13 27/30 Read block 131 len 36864 28/30 Read block 538 len 32768 29/30 Write block 397 len 20480 data: 11 13 15 16 18 30/30 Write block 235 len 32768 data: 13 14 17 19 19 19 19 21 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:33.095209916Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1891} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 314 314 314 T T T 1 020-039 175 175 175 315 315 315 T T T 2 040-059 174 174 174 314 314 314 F F F 3 060-079 176 176 176 317 317 317 F F F 4 080-099 169 169 169 305 305 305 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 172 172 172 311 311 311 F F F 7 140-159 172 172 172 311 311 311 F F F 8 160-179 172 172 172 311 311 311 F F F 9 180-199 176 176 176 316 316 316 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 176 176 176 317 317 317 T T T 12 240-259 176 176 176 317 317 317 T T T 13 260-279 175 175 175 315 315 315 F F F 14 280-299 176 176 176 316 316 316 F F F 15 300-319 176 176 176 317 317 317 F F F 16 320-339 174 174 174 314 314 314 F F F 17 340-359 174 174 174 314 314 314 F F F 18 360-379 169 169 169 305 305 305 F F F 19 380-399 168 168 168 304 304 304 T T T 20 400-419 174 174 174 314 314 314 T T T 21 420-439 175 175 175 315 315 315 F F F 22 440-459 169 169 169 305 305 305 T T T 23 460-479 174 174 174 313 313 313 F F F 24 480-499 173 173 173 312 312 312 F F F 25 500-519 175 175 175 315 315 315 F F F 26 520-539 175 175 175 315 315 315 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 176 176 176 317 317 317 F F F 29 580-599 176 176 176 317 317 317 F F F Max gen: 176, Max flush: 317 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 177 177 177 318 318 318 F F F 1 020-039 177 177 177 318 318 318 F F F 2 040-059 174 174 174 314 314 314 F F F 3 060-079 176 176 176 317 317 317 F F F 4 080-099 169 169 169 305 305 305 F F F 5 100-119 172 172 172 310 310 310 F F F 6 120-139 172 172 172 311 311 311 F F F 7 140-159 172 172 172 311 311 311 F F F 8 160-179 172 172 172 311 311 311 F F F 9 180-199 176 176 176 316 316 316 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 177 177 177 318 318 318 F F F 12 240-259 177 177 177 318 318 318 F F F 13 260-279 175 175 175 315 315 315 F F F 14 280-299 176 176 176 316 316 316 F F F 15 300-319 176 176 176 317 317 317 F F F 16 320-339 174 174 174 314 314 314 F F F 17 340-359 174 174 174 314 314 314 F F F 18 360-379 169 169 169 305 305 305 F F F 19 380-399 177 177 177 318 318 318 F F F 20 400-419 177 177 177 318 318 318 F F F 21 420-439 175 175 175 315 315 315 F F F 22 440-459 177 177 177 318 318 318 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 173 173 173 312 312 312 F F F 25 500-519 175 175 175 315 315 315 F F F 26 520-539 175 175 175 315 315 315 F F F 27 540-559 167 167 167 303 303 303 F F F 28 560-579 176 176 176 317 317 317 F F F 29 580-599 176 176 176 317 317 317 F F F Max gen: 177, Max flush: 318 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.344173062Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:37.386377932Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.386759055Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.417641598Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.418230131Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.418263747Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.418282286Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.418297214Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899} {"msg":"Crucible cba4e9a1-0fbf-437e-8ab4-88eb078289e6 has session id: 0d26f7d4-4b03-4c78-b815-2f0543b62eb8","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.418902645Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Upstairs opts: Upstairs UUID: cba4e9a1-0fbf-437e-8ab4-88eb078289e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.418947541Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Crucible stats registered with UUID: cba4e9a1-0fbf-437e-8ab4-88eb078289e6","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.418971719Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:37.418994007Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419393628Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419426094Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899} {"msg":"cba4e9a1-0fbf-437e-8ab4-88eb078289e6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419480249Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419506997Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"0","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419533544Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"0","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419558892Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"1","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419586209Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"1","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419616776Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"2","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419643333Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"2","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419668721Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419684129Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419698368Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:52337)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419722556Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:34643)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419749853Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41722)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.419908207Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.420432327Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"2","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.420459454Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"0","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.420474102Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"1","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421050076Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"1","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421090162Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"2","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421156776Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"0","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421615461Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421644178Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"[0]R flush_numbers[0..12]: [318, 318, 314, 317, 305, 310, 311, 311, 311, 316, 317, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421669206Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"[0]R generation[0..12]: [177, 177, 174, 176, 169, 172, 172, 172, 172, 176, 176, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421684134Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421698193Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"[1]R flush_numbers[0..12]: [318, 318, 314, 317, 305, 310, 311, 311, 311, 316, 317, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421711842Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"[1]R generation[0..12]: [177, 177, 174, 176, 169, 172, 172, 172, 172, 176, 176, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421725281Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421742969Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"[2]R flush_numbers[0..12]: [318, 318, 314, 317, 305, 310, 311, 311, 311, 316, 317, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421767716Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"[2]R generation[0..12]: [177, 177, 174, 176, 169, 172, 172, 172, 172, 176, 176, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421795304Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421816982Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Max found gen is 178","v"Wait for a query_work_queue command to finish before sending IO :0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421846489Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421871936Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421886505Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421899204Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"":"downstairs","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421920982Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421943499Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"0","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421968887Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"1","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.421987125Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"client":"2","":"downstairs","session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.422000344Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.422020542Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"cba4e9a1-0fbf-437e-8ab4-88eb078289e6 is now active with session: 0d26f7d4-4b03-4c78-b815-2f0543b62eb8","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.422044039Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.422066827Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899,"session_id":"0d26f7d4-4b03-4c78-b815-2f0543b62eb8"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:37.422083816Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 428 len 12288 data: 14 16 18 02/30 Read block 378 len 8192 03/30 Read block 140 len 20480 04/30 Read block 309 len 24576 05/30 Read block 223 len 36864 06/30 Read block 68 len 8192 07/30 Write block 26 len 24576 data: 14 12 12 13 13 13 08/30 Write block 30 len 40960 data: 14 14 13 14 15 13 12 12 14 11 09/30 Read block 49 len 32768 10/30 Read block 592 len 24576 11/30 Flush 12/30 Write block 274 len 16384 data: 16 16 15 15 13/30 Flush 14/30 Write block 334 len 32768 data: 15 14 14 16 14 14 13 13 15/30 Write block 109 len 12288 data: 14 14 11 16/30 Read block 307 len 36864 17/30 Read block 145 len 20480 18/30 Read block 170 len 8192 19/30 Read block 526 len 20480 20/30 Write block 130 len 4096 data: 8 21/30 Read block 366 len 4096 22/30 Write block 489 len 16384 data: 13 12 11 15 23/30 Write block 332 len 32768 data: 10 13 16 15 15 17 15 15 24/30 Write block 525 len 4096 data: 12 25/30 Write block 551 len 20480 data: 14 15 15 17 18 26/30 Flush 27/30 Write block 285 len 4096 data: 10 28/30 Write block 492 len 36864 data: 16 18 18 17 17 14 17 18 17 29/30 Write block 12 len 40960 data: 15 16 16 17 16 12 12 15 14 11 30/30 Read block 228 len 16384 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 4 3 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:37.440886595Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1899} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 177 177 177 318 318 318 T T T 1 020-039 178 178 178 319 319 319 T T T 2 040-059 174 174 174 314 314 314 F F F 3 060-079 176 176 176 317 317 317 F F F 4 080-099 169 169 169 305 305 305 F F F 5 100-119 178 178 178 321 321 321 F F F 6 120-139 178 178 178 321 321 321 F F F 7 140-159 172 172 172 311 311 311 F F F 8 160-179 172 172 172 311 311 311 F F F 9 180-199 176 176 176 316 316 316 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 177 177 177 318 318 318 F F F 12 240-259 177 177 177 318 318 318 F F F 13 260-279 178 178 178 320 320 320 F F F 14 280-299 176 176 176 316 316 316 T T T 15 300-319 176 176 176 317 317 317 F F F 16 320-339 178 178 178 321 321 321 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 169 169 169 305 305 305 F F F 19 380-399 177 177 177 318 318 318 F F F 20 400-419 177 177 177 318 318 318 F F F 21 420-439 178 178 178 319 319 319 F F F 22 440-459 177 177 177 318 318 318 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 178 178 178 321 321 321 T T T 25 500-519 175 175 175 315 315 315 T T T 26 520-539 178 178 178 321 321 321 F F F 27 540-559 178 178 178 321 321 321 F F F 28 560-579 176 176 176 317 317 317 F F F 29 580-599 176 176 176 317 317 317 F F F Max gen: 178, Max flush: 321 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 179 179 179 322 322 322 F F F 2 040-059 174 174 174 314 314 314 F F F 3 060-079 176 176 176 317 317 317 F F F 4 080-099 169 169 169 305 305 305 F F F 5 100-119 178 178 178 321 321 321 F F F 6 120-139 178 178 178 321 321 321 F F F 7 140-159 172 172 172 311 311 311 F F F 8 160-179 172 172 172 311 311 311 F F F 9 180-199 176 176 176 316 316 316 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 177 177 177 318 318 318 F F F 12 240-259 177 177 177 318 318 318 F F F 13 260-279 178 178 178 320 320 320 F F F 14 280-299 179 179 179 322 322 322 F F F 15 300-319 176 176 176 317 317 317 F F F 16 320-339 178 178 178 321 321 321 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 169 169 169 305 305 305 F F F 19 380-399 177 177 177 318 318 318 F F F 20 400-419 177 177 177 318 318 318 F F F 21 420-439 178 178 178 319 319 319 F F F 22 440-459 177 177 177 318 318 318 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 179 179 179 322 322 322 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 178 178 178 321 321 321 F F F 27 540-559 178 178 178 321 321 321 F F F 28 560-579 176 176 176 317 317 317 F F F 29 580-599 176 176 176 317 317 317 F F F Max gen: 179, Max flush: 322 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.702886512Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.746277377Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.746911125Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.746976589Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.747021844Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.747043142Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907} {"msg":"Crucible 07ed6f5e-38bf-4658-be7d-491504801db3 has session id: d6c3e5ca-7f7f-4c85-86c9-d88dc6859239","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.747568661Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"Upstairs opts: Upstairs UUID: 07ed6f5e-38bf-4658-be7d-491504801db3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.747611047Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"Crucible stats registered with UUID: 07ed6f5e-38bf-4658-be7d-491504801db3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.747633295Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:41.747653493Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748119158Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748146255Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907} {"msg":"07ed6f5e-38bf-4658-be7d-491504801db3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748204279Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748227737Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"0","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748251185Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"0","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748264923Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"1","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748355225Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"1","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748388921Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"2","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748418728Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"2","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748448945Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748559785Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748596111Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"ds_connection connected from Ok(127.0.0.1:36358)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748622619Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"ds_connection connected from Ok(127.0.0.1:58485)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748643047Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"ds_connection connected from Ok(127.0.0.1:63727)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.748657365Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.749196633Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"0","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.74922637Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"1","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.749311572Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"2","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.749831141Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"0","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.749980476Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"1","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750028292Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"2","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750311134Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750549401Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"[0]R flush_numbers[0..12]: [322, 322, 314, 317, 305, 321, 321, 311, 311, 316, 317, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750580658Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"[0]R generation[0..12]: [179, 179, 174, 176, 169, 178, 178, 172, 172, 176, 176, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750597686Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750612205Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"[1]R flush_numbers[0..12]: [322, 322, 314, 317, 305, 321, 321, 311, 311, 316, 317, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750626624Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"[1]R generation[0..12]: [179, 179, 174, 176, 169, 178, 178, 172, 172, 176, 176, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750640672Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750654521Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"[2]R flush_numbers[0..12]: [322, 322, 314, 317, 305, 321, 321, 311, 311, 316, 317, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.75066778Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"[2]R generation[0..12]: [179, 179, 174, 176, 169, 178, 178, 172, 172, 176, 176, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750681338Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750699266Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750712485Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750725274Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750745732Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750769679Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"":"downstairs","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750789818Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750811435Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"0","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750828374Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"1","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750843732Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"client":"2","":"downstairs","session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-01-28T17:16:41.75086716Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750886648Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"07ed6f5e-38bf-4658-be7d-491504801db3 is now active with session: d6c3e5ca-7f7f-4c85-86c9-d88dc6859239","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750899497Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750911476Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907,"session_id":"d6c3e5ca-7f7f-4c85-86c9-d88dc6859239"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:41.750923515Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 452 len 4096 data: 12 02/30 Flush 03/30 Write block 248 len 12288 data: 17 13 15 04/30 Write block 545 len 32768 data: 14 14 11 11 13 14 15 16 05/30 Flush 06/30 Flush 07/30 Flush 08/30 Write block 373 len 40960 data: 8 9 10 12 11 12 11 11 11 13 09/30 Write block 480 len 20480 data: 14 13 13 12 9 10/30 Write block 29 len 8192 data: 14 15 11/30 Write block 288 len 12288 data: 8 11 12 12/30 Read block 254 len 20480 13/30 Read block 99 len 12288 14/30 Read block 548 len 20480 15/30 Read block 201 len 24576 16/30 Write block 282 len 20480 data: 10 9 7 11 9 17/30 Read block 144 len 16384 18/30 Read block 413 len 12288 19/30 Write block 87 len 40960 data: 14 16 17 17 16 14 14 11 12 14 20/30 Read block 312 len 4096 21/30 Read block 143 len 36864 22/30 Write block 298 len 24576 data: 14 12 15 17 14 15 23/30 Flush 24/30 Write block 327 len 28672 data: 15 16 16 16 12 11 14 25/30 Read block 489 len 36864 26/30 Read block 83 len 36864 27/30 Read block 562 len 8192 28/30 Read block 230 len 16384 29/30 Write block 261 len 24576 data: 14 17 15 13 18 19 30/30 Write block 259 len 24576 data: 12 11 15 18 16 14 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:41.76976613Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1907} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 180 180 180 327 327 327 F F F 2 040-059 174 174 174 314 314 314 F F F 3 060-079 176 176 176 317 317 317 F F F 4 080-099 180 180 180 327 327 327 F F F 5 100-119 178 178 178 321 321 321 F F F 6 120-139 178 178 178 321 321 321 F F F 7 140-159 172 172 172 311 311 311 F F F 8 160-179 172 172 172 311 311 311 F F F 9 180-199 176 176 176 316 316 316 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 177 177 177 318 318 318 F F F 12 240-259 180 180 180 324 324 324 T T T 13 260-279 178 178 178 320 320 320 T T T 14 280-299 180 180 180 327 327 327 F F F 15 300-319 180 180 180 327 327 327 F F F 16 320-339 178 178 178 321 321 321 T T T 17 340-359 178 178 178 321 321 321 F F F 18 360-379 180 180 180 327 327 327 F F F 19 380-399 180 180 180 327 327 327 F F F 20 400-419 177 177 177 318 318 318 F F F 21 420-439 178 178 178 319 319 319 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 180 180 180 327 327 327 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 178 178 178 321 321 321 F F F 27 540-559 180 180 180 324 324 324 F F F 28 560-579 176 176 176 317 317 317 F F F 29 580-599 176 176 176 317 317 317 F F F Max gen: 180, Max flush: 327 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 180 180 180 327 327 327 F F F 2 040-059 174 174 174 314 314 314 F F F 3 060-079 176 176 176 317 317 317 F F F 4 080-099 180 180 180 327 327 327 F F F 5 100-119 178 178 178 321 321 321 F F F 6 120-139 178 178 178 321 321 321 F F F 7 140-159 172 172 172 311 311 311 F F F 8 160-179 172 172 172 311 311 311 F F F 9 180-199 176 176 176 316 316 316 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 177 177 177 318 318 318 F F F 12 240-259 181 181 181 328 328 328 F F F 13 260-279 181 181 181 328 328 328 F F F 14 280-299 180 180 180 327 327 327 F F F 15 300-319 180 180 180 327 327 327 F F F 16 320-339 181 181 181 328 328 328 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 180 180 180 327 327 327 F F F 19 380-399 180 180 180 327 327 327 F F F 20 400-419 177 177 177 318 318 318 F F F 21 420-439 178 178 178 319 319 319 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 180 180 180 327 327 327 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 178 178 178 321 321 321 F F F 27 540-559 180 180 180 324 324 324 F F F 28 560-579 176 176 176 317 317 317 F F F 29 580-599 176 176 176 317 317 317 F F F Max gen: 181, Max flush: 328 Begin loop 91 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:45.995589469Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.039171316Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.039813513Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.039854679Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.039870088Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.039884526Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915} {"msg":"Crucible ab2d92b4-4ec6-4e9e-a896-9dc7f915264f has session id: cb3c13e0-85d2-4b33-ad58-5320a0bc4b23","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.040483368Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Upstairs opts: Upstairs UUID: ab2d92b4-4ec6-4e9e-a896-9dc7f915264f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.040534983Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Crucible stats registered with UUID: ab2d92b4-4ec6-4e9e-a896-9dc7f915264f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.04056632Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:46.040594057Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041104247Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041130265Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915} {"msg":"ab2d92b4-4ec6-4e9e-a896-9dc7f915264f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041187819Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041214057Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"0","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041237235Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"0","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041251473Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"1","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041265652Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"1","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.04127879Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"2","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041296579Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"2","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041320956Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041347464Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041373901Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"ds_connection connected from Ok(127.0.0.1:56248)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041401388Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"ds_connection connected from Ok(127.0.0.1:64905)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041432145Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"ds_connection connected from Ok(127.0.0.1:52241)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.041459313Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.042127648Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"1","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.042167484Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"0","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.042215959Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"2","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.04272637Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"0","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.042867706Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"1","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.042896973Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"2","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.04333508Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043451609Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"[0]R flush_numbers[0..12]: [322, 327, 314, 317, 327, 321, 321, 311, 311, 316, 317, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043515693Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"[0]R generation[0..12]: [179, 180, 174, 176, 180, 178, 178, 172, 172, 176, 176, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043535641Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043560898Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"[1]R flush_numbers[0..12]: [322, 327, 314, 317, 327, 321, 321, 311, 311, 316, 317, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043575177Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"[1]R generation[0..12]: [179, 180, 174, 176, 180, 178, 178, 172, 172, 176, 176, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043592635Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043607234Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"[2]R flush_numbers[0..12]: [322, 327, 314, 317, 327, 321, 321, 311, 311, 316, 317, 318]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043620702Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"[2]R generation[0..12]: [179, 180, 174, 176, 180, 178, 178, 172, 172, 176, 176, 177]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043635221Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043649029Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043661788Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043674857Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:16:46.043687686Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043705874Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"":"downstairs","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043718833Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043730632Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"0","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.04374369Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"1","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043756739Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"client":"2","":"downstairs","session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043769658Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043781777Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"ab2d92b4-4ec6-4e9e-a896-9dc7f915264f is now active with session: cb3c13e0-85d2-4b33-ad58-5320a0bc4b23","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043796455Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043808324Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915,"session_id":"cb3c13e0-85d2-4b33-ad58-5320a0bc4b23"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:46.043820363Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 73 len 36864 data: 16 15 14 12 11 12 13 13 14 02/30 Flush 03/30 Write block 270 len 4096 data: 15 04/30 Read block 334 len 8192 05/30 Read block 493 len 12288 06/30 Write block 165 len 40960 data: 18 15 15 14 13 13 14 17 18 17 07/30 Read block 214 len 12288 08/30 Write block 579 len 8192 data: 15 15 09/30 Read block 199 len 20480 10/30 Read block 477 len 28672 11/30 Read block 174 len 36864 12/30 Read block 254 len 8192 13/30 Read block 388 len 4096 14/30 Read block 259 len 24576 15/30 Write block 599 len 4096 data: 4 16/30 Read block 122 len 16384 17/30 Read block 67 len 24576 18/30 Read block 172 len 40960 19/30 Write block 156 len 36864 data: 9 9 17 15 16 16 17 19 24 20/30 Read block 328 len 16384 21/30 Write block 102 len 12288 data: 10 8 12 22/30 Read block 105 len 24576 23/30 Write block 330 len 36864 data: 17 13 12 15 17 16 16 18 16 24/30 Write block 236 len 8192 data: 15 18 25/30 Read block 441 len 36864 26/30 Write block 34 len 24576 data: 16 14 13 13 15 12 27/30 Write block 583 len 16384 data: 21 22 24 15 28/30 Read block 369 len 8192 29/30 Write block 70 len 12288 data: 12 15 17 30/30 Write block 296 len 20480 data: 13 13 15 13 16 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 1 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:46.062694415Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1915} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 180 180 180 327 327 327 T T T 2 040-059 174 174 174 314 314 314 F F F 3 060-079 182 182 182 329 329 329 T T T 4 080-099 182 182 182 329 329 329 F F F 5 100-119 178 178 178 321 321 321 T T T 6 120-139 178 178 178 321 321 321 F F F 7 140-159 172 172 172 311 311 311 T T T 8 160-179 172 172 172 311 311 311 T T T 9 180-199 176 176 176 316 316 316 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 177 177 177 318 318 318 T T T 12 240-259 181 181 181 328 328 328 F F F 13 260-279 181 181 181 328 328 328 T T T 14 280-299 180 180 180 327 327 327 T T T 15 300-319 180 180 180 327 327 327 T T T 16 320-339 181 181 181 328 328 328 T T T 17 340-359 178 178 178 321 321 321 F F F 18 360-379 180 180 180 327 327 327 F F F 19 380-399 180 180 180 327 327 327 F F F 20 400-419 177 177 177 318 318 318 F F F 21 420-439 178 178 178 319 319 319 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 180 180 180 327 327 327 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 178 178 178 321 321 321 F F F 27 540-559 180 180 180 324 324 324 F F F 28 560-579 176 176 176 317 317 317 T T T 29 580-599 176 176 176 317 317 317 T T T Max gen: 182, Max flush: 329 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 183 183 183 330 330 330 F F F 2 040-059 174 174 174 314 314 314 F F F 3 060-079 183 183 183 330 330 330 F F F 4 080-099 182 182 182 329 329 329 F F F 5 100-119 183 183 183 330 330 330 F F F 6 120-139 178 178 178 321 321 321 F F F 7 140-159 183 183 183 330 330 330 F F F 8 160-179 183 183 183 330 330 330 F F F 9 180-199 176 176 176 316 316 316 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 183 183 183 330 330 330 F F F 12 240-259 181 181 181 328 328 328 F F F 13 260-279 183 183 183 330 330 330 F F F 14 280-299 183 183 183 330 330 330 F F F 15 300-319 183 183 183 330 330 330 F F F 16 320-339 183 183 183 330 330 330 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 180 180 180 327 327 327 F F F 19 380-399 180 180 180 327 327 327 F F F 20 400-419 177 177 177 318 318 318 F F F 21 420-439 178 178 178 319 319 319 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 180 180 180 327 327 327 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 178 178 178 321 321 321 F F F 27 540-559 180 180 180 324 324 324 F F F 28 560-579 183 183 183 330 330 330 F F F 29 580-599 183 183 183 330 330 330 F F F Max gen: 183, Max flush: 330 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.34382349Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.387249701Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.387899178Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.387938824Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.387953643Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.387968122Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923} {"msg":"Crucible 19e34a5e-5aec-448f-9f19-dce2fc340061 has session id: 9522aefd-4c5a-4e40-891a-12b9fdb17e44","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.3884925Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Upstairs opts: Upstairs UUID: 19e34a5e-5aec-448f-9f19-dce2fc340061, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.388538686Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Crucible stats registered with UUID: 19e34a5e-5aec-448f-9f19-dce2fc340061","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.388562304Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:50.388582922Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.389005581Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.389030598Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923} {"msg":"19e34a5e-5aec-448f-9f19-dce2fc340061 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.3891072Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.389127239Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"0","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.389156646Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"0","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.389171054Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"1","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.389194042Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"1","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.3892184Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"2","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.389242767Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"2","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.389266535Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.389290413Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.38931388Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"ds_connection connected from Ok(127.0.0.1:58627)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.389356616Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"ds_connection connected from Ok(127.0.0.1:58169)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.389383764Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"ds_connection connected from Ok(127.0.0.1:62118)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.389408191Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.390157708Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"2","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.390196065Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"1","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.390223032Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"0","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.390873209Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"1","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.390910965Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"2","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.390937262Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"0","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.39148034Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391506237Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"[0]R flush_numbers[0..12]: [322, 330, 314, 330, 329, 330, 321, 330, 330, 316, 317, 330]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391533564Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"[0]R generation[0..12]: [179, 183, 174, 183, 182, 183, 178, 183, 183, 176, 176, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391549423Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391563911Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"[1]R flush_numbers[0..12]: [322, 330, 314, 330, 329, 330, 321, 330, 330, 316, 317, 330]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.39157735Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"[1]R generation[0..12]: [179, 183, 174, 183, 182, 183, 178, 183, 183, 176, 176, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391591019Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391604737Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"[2]R flush_numbers[0..12]: [322, 330, 314, 330, 329, 330, 321, 330, 330, 316, 317, 330]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391618166Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"[2]R generation[0..12]: [179, 183, 174, 183, 182, 183, 178, 183, 183, 176, 176, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391632285Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391645793Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391662872Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.39167687Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391689609Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391702168Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391723496Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391735175Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"0","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391748143Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"1","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391760892Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"client":"2","":"downstairs","session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391773681Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.39178561Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"19e34a5e-5aec-448f-9f19-dce2fc340061 is now active with session: 9522aefd-4c5a-4e40-891a-12b9fdb17e44","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391798109Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391809847Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923,"session_id":"9522aefd-4c5a-4e40-891a-12b9fdb17e44"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:50.391821626Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 368 len 28672 data: 10 11 11 12 11 9 10 02/30 Write block 398 len 4096 data: 14 03/30 Write block 170 len 20480 data: 14 15 18 19 18 04/30 Read block 435 len 16384 05/30 Read block 427 len 16384 06/30 Read block 494 len 20480 07/30 Write block 391 len 24576 data: 14 15 14 10 9 9 08/30 Write block 358 len 32768 data: 16 15 12 11 8 7 10 10 09/30 Write block 564 len 4096 data: 8 10/30 Read block 407 len 4096 11/30 Read block 488 len 40960 12/30 Read block 472 len 20480 13/30 Read block 332 len 28672 14/30 Write block 414 len 40960 data: 11 11 12 11 10 11 13 13 12 12 15/30 Read block 565 len 40960 16/30 Write block 30 len 20480 data: 16 15 14 15 17 17/30 Read block 239 len 16384 18/30 Write block 392 len 40960 data: 16 15 11 10 10 12 15 16 17 19 19/30 Write block 400 len 28672 data: 18 20 17 21 17 17 15 20/30 Read block 14 len 20480 21/30 Flush 22/30 Write block 161 len 16384 data: 17 18 20 25 23/30 Read block 169 len 16384 24/30 Write block 428 len 40960 data: 15 17 19 17 15 18 22 19 17 18 25/30 Flush 26/30 Write block 272 len 24576 data: 14 13 17 17 16 16 27/30 Read block 512 len 20480 28/30 Write block 197 len 8192 data: 15 16 29/30 Read block 493 len 20480 30/30 Write block 254 len 8192 data: 15 12 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 5 Done Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 3 2 6 Done 4 2 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:50.411621488Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1923} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 184 184 184 331 331 331 F F F 2 040-059 174 174 174 314 314 314 F F F 3 060-079 183 183 183 330 330 330 F F F 4 080-099 182 182 182 329 329 329 F F F 5 100-119 183 183 183 330 330 330 F F F 6 120-139 178 178 178 321 321 321 F F F 7 140-159 183 183 183 330 330 330 F F F 8 160-179 184 184 184 332 332 332 F F F 9 180-199 176 176 176 316 316 316 T T T 10 200-219 176 176 176 317 317 317 F F F 11 220-239 183 183 183 330 330 330 F F F 12 240-259 181 181 181 328 328 328 T T T 13 260-279 183 183 183 330 330 330 T T T 14 280-299 183 183 183 330 330 330 F F F 15 300-319 183 183 183 330 330 330 F F F 16 320-339 183 183 183 330 330 330 F F F 17 340-359 184 184 184 331 331 331 F F F 18 360-379 184 184 184 331 331 331 F F F 19 380-399 184 184 184 331 331 331 F F F 20 400-419 184 184 184 331 331 331 F F F 21 420-439 184 184 184 332 332 332 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 180 180 180 327 327 327 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 178 178 178 321 321 321 F F F 27 540-559 180 180 180 324 324 324 F F F 28 560-579 184 184 184 331 331 331 F F F 29 580-599 183 183 183 330 330 330 F F F Max gen: 184, Max flush: 332 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 184 184 184 331 331 331 F F F 2 040-059 174 174 174 314 314 314 F F F 3 060-079 183 183 183 330 330 330 F F F 4 080-099 182 182 182 329 329 329 F F F 5 100-119 183 183 183 330 330 330 F F F 6 120-139 178 178 178 321 321 321 F F F 7 140-159 183 183 183 330 330 330 F F F 8 160-179 184 184 184 332 332 332 F F F 9 180-199 185 185 185 333 333 333 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 183 183 183 330 330 330 F F F 12 240-259 185 185 185 333 333 333 F F F 13 260-279 185 185 185 333 333 333 F F F 14 280-299 183 183 183 330 330 330 F F F 15 300-319 183 183 183 330 330 330 F F F 16 320-339 183 183 183 330 330 330 F F F 17 340-359 184 184 184 331 331 331 F F F 18 360-379 184 184 184 331 331 331 F F F 19 380-399 184 184 184 331 331 331 F F F 20 400-419 184 184 184 331 331 331 F F F 21 420-439 184 184 184 332 332 332 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 180 180 180 327 327 327 F F F 25 500-519 179 179 179 322 322 322 F F F 26 520-539 178 178 178 321 321 321 F F F 27 540-559 180 180 180 324 324 324 F F F 28 560-579 184 184 184 331 331 331 F F F 29 580-599 183 183 183 330 330 330 F F F Max gen: 185, Max flush: 333 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.637499673Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:54.680008094Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.680397276Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.711929346Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.712491821Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.712526527Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.712541156Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.712555594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931} {"msg":"Crucible cf61e9be-471d-4122-8548-28800fe6357d has session id: acd69cfe-9db7-4765-a312-65b3998162b7","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713091812Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Upstairs opts: Upstairs UUID: cf61e9be-471d-4122-8548-28800fe6357d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713133358Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Crucible stats registered with UUID: cf61e9be-471d-4122-8548-28800fe6357d","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713150227Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:54.713163075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713500862Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.71352421Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931} {"msg":"cf61e9be-471d-4122-8548-28800fe6357d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713596333Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713614981Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"0","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713636269Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"0","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713650078Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"1","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713664286Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"1","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713682455Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"2","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713708282Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"2","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.71373283Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713757968Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713785415Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:39792)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713846739Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34949)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713891135Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53472)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.713920282Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.714602305Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"1","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.714642741Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"2","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.714675818Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"0","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715208426Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"1","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715236353Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"0","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715252512Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"2","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715770381Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715795179Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"[0]R flush_numbers[0..12]: [322, 331, 314, 330, 329, 330, 321, 330, 332, 333, 317, 330]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715821856Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"[0]R generation[0..12]: [179, 184, 174, 183, 182, 183, 178, 183, 184, 185, 176, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715836685Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715850904Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"[1]R flush_numbers[0..12]: [322, 331, 314, 330, 329, 330, 321, 330, 332, 333, 317, 330]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715867862Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"[1]R generation[0..12]: [179, 184, 174, 183, 182, 183, 178, 183, 184, 185, 176, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715881851Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715895479Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"[2]R flush_numbers[0..12]: [322, 331, 314, 330, 329, 330, 321, 330, 332, 333, 317, 330]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715909068Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"[2]R generation[0..12]: [179, 184, 174, 183, 182, 183, 178, 183, 184, 185, 176, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715922277Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715935905Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715948594Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Generation requested: 186 >= found:186","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-01-28T17:16:54.715967382Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.715980281Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.716003699Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"":"downstairs","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.716024927Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.716038685Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"0","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.716052264Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"1","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.716064973Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"client":"2","":"downstairs","session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.716080401Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.71609707Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"cf61e9be-471d-4122-8548-28800fe6357d is now active with session: acd69cfe-9db7-4765-a312-65b3998162b7","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.716119598Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.716142435Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931,"session_id":"acd69cfe-9db7-4765-a312-65b3998162b7"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:54.716166143Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 263 len 20480 data: 17 15 19 20 20 02/30 Flush 03/30 Write block 549 len 40960 data: 14 15 16 17 16 18 19 16 16 18 04/30 Read block 391 len 20480 05/30 Write block 287 len 32768 data: 9 9 12 13 11 12 13 15 06/30 Read block 293 len 36864 07/30 Read block 63 len 32768 08/30 Read block 224 len 16384 09/30 Write block 112 len 12288 data: 11 15 16 10/30 Read block 110 len 32768 11/30 Write block 38 len 20480 data: 16 13 12 11 11 12/30 Write block 577 len 40960 data: 18 20 16 16 16 21 22 23 25 16 13/30 Write block 211 len 24576 data: 16 12 15 15 15 15 14/30 Write block 393 len 12288 data: 16 12 11 15/30 Read block 542 len 36864 16/30 Read block 148 len 20480 17/30 Flush 18/30 Write block 319 len 32768 data: 9 12 13 17 15 17 16 14 19/30 Read block 511 len 36864 20/30 Read block 68 len 20480 21/30 Read block 34 len 16384 22/30 Write block 270 len 16384 data: 16 16 15 14 23/30 Read block 491 len 36864 24/30 Flush 25/30 Write block 37 len 8192 data: 14 17 26/30 Read block 133 len 28672 27/30 Read block 55 len 8192 28/30 Write block 498 len 36864 data: 18 19 18 15 15 19 18 17 16 29/30 Write block 115 len 40960 data: 15 13 11 8 9 8 8 7 6 5 30/30 Write block 240 len 36864 data: 20 20 22 17 17 16 14 16 18 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:54.736254877Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1931} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 186 186 186 335 335 335 T T T 2 040-059 186 186 186 335 335 335 F F F 3 060-079 183 183 183 330 330 330 F F F 4 080-099 182 182 182 329 329 329 F F F 5 100-119 186 186 186 335 335 335 T T T 6 120-139 178 178 178 321 321 321 T T T 7 140-159 183 183 183 330 330 330 F F F 8 160-179 184 184 184 332 332 332 F F F 9 180-199 185 185 185 333 333 333 F F F 10 200-219 186 186 186 335 335 335 F F F 11 220-239 183 183 183 330 330 330 F F F 12 240-259 185 185 185 333 333 333 T T T 13 260-279 186 186 186 336 336 336 F F F 14 280-299 186 186 186 335 335 335 F F F 15 300-319 186 186 186 336 336 336 F F F 16 320-339 186 186 186 336 336 336 F F F 17 340-359 184 184 184 331 331 331 F F F 18 360-379 184 184 184 331 331 331 F F F 19 380-399 186 186 186 335 335 335 F F F 20 400-419 184 184 184 331 331 331 F F F 21 420-439 184 184 184 332 332 332 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 180 180 180 327 327 327 T T T 25 500-519 179 179 179 322 322 322 T T T 26 520-539 178 178 178 321 321 321 F F F 27 540-559 186 186 186 335 335 335 F F F 28 560-579 186 186 186 335 335 335 F F F 29 580-599 186 186 186 335 335 335 F F F Max gen: 186, Max flush: 336 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 187 187 187 337 337 337 F F F 2 040-059 186 186 186 335 335 335 F F F 3 060-079 183 183 183 330 330 330 F F F 4 080-099 182 182 182 329 329 329 F F F 5 100-119 187 187 187 337 337 337 F F F 6 120-139 187 187 187 337 337 337 F F F 7 140-159 183 183 183 330 330 330 F F F 8 160-179 184 184 184 332 332 332 F F F 9 180-199 185 185 185 333 333 333 F F F 10 200-219 186 186 186 335 335 335 F F F 11 220-239 183 183 183 330 330 330 F F F 12 240-259 187 187 187 337 337 337 F F F 13 260-279 186 186 186 336 336 336 F F F 14 280-299 186 186 186 335 335 335 F F F 15 300-319 186 186 186 336 336 336 F F F 16 320-339 186 186 186 336 336 336 F F F 17 340-359 184 184 184 331 331 331 F F F 18 360-379 184 184 184 331 331 331 F F F 19 380-399 186 186 186 335 335 335 F F F 20 400-419 184 184 184 331 331 331 F F F 21 420-439 184 184 184 332 332 332 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 187 187 187 337 337 337 F F F 25 500-519 187 187 187 337 337 337 F F F 26 520-539 178 178 178 321 321 321 F F F 27 540-559 186 186 186 335 335 335 F F F 28 560-579 186 186 186 335 335 335 F F F 29 580-599 186 186 186 335 335 335 F F F Max gen: 187, Max flush: 337 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.011223072Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.054750614Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.05540198Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.055454775Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.055473873Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.055493081Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939} {"msg":"Crucible 0a55ddf8-6f95-4466-a5a1-d2e5b5a08844 has session id: 79741b6a-22f9-4b44-a977-5733f31bf253","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.055964496Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"Upstairs opts: Upstairs UUID: 0a55ddf8-6f95-4466-a5a1-d2e5b5a08844, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056106472Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"Crucible stats registered with UUID: 0a55ddf8-6f95-4466-a5a1-d2e5b5a08844","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.05612888Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:59.056142218Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056642739Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056672207Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939} {"msg":"0a55ddf8-6f95-4466-a5a1-d2e5b5a08844 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056705583Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056726241Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"0","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056765578Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"0","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056793725Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"1","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056823672Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"1","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.05684633Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"2","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056873507Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"2","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056898914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056926942Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056954489Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"ds_connection connected from Ok(127.0.0.1:50127)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.056980037Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"ds_connection connected from Ok(127.0.0.1:63514)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.057004104Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"ds_connection connected from Ok(127.0.0.1:36411)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.057027042Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.057656781Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"0","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.057683608Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"1","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.057698077Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"2","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.058321706Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"0","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.058351253Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"1","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.05837795Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"2","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.058931827Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.058973852Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"[0]R flush_numbers[0..12]: [322, 337, 335, 330, 329, 337, 337, 330, 332, 333, 335, 330]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059002829Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"[0]R generation[0..12]: [179, 187, 186, 183, 182, 187, 187, 183, 184, 185, 186, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059022738Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059040756Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"[1]R flush_numbers[0..12]: [322, 337, 335, 330, 329, 337, 337, 330, 332, 333, 335, 330]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059057724Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"[1]R generation[0..12]: [179, 187, 186, 183, 182, 187, 187, 183, 184, 185, 186, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059074393Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059091351Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"[2]R flush_numbers[0..12]: [322, 337, 335, 330, 329, 337, 337, 330, 332, 333, 335, 330]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059108319Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"[2]R generation[0..12]: [179, 187, 186, 183, 182, 187, 187, 183, 184, 185, 186, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059125078Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059142156Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059158094Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059180432Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"Next flush: 338","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.05920101Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059217349Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"":"downstairs","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059233797Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059248606Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"0","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059265034Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"1","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059281173Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"client":"2","":"downstairs","session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059297421Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.05931245Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"0a55ddf8-6f95-4466-a5a1-d2e5b5a08844 is now active with session: 79741b6a-22f9-4b44-a977-5733f31bf253","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059328198Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059343076Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939,"session_id":"79741b6a-22f9-4b44-a977-5733f31bf253"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:16:59.059358185Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 87 len 24576 data: 15 17 18 18 17 15 03/30 Read block 525 len 20480 04/30 Write block 216 len 28672 data: 16 11 10 11 12 13 17 05/30 Write block 268 len 24576 data: 16 14 17 17 16 15 06/30 Read block 140 len 40960 07/30 Read block 200 len 4096 08/30 Read block 363 len 24576 09/30 Read block 6 len 32768 10/30 Write block 566 len 20480 data: 8 9 12 13 15 11/30 Flush 12/30 Write block 188 len 20480 data: 10 10 11 14 15 13/30 Write block 107 len 20480 data: 14 14 15 15 12 14/30 Write block 484 len 40960 data: 10 9 11 15 12 14 13 12 17 19 15/30 Write block 542 len 32768 data: 13 12 11 15 15 12 12 15 16/30 Read block 5 len 8192 17/30 Write block 114 len 28672 data: 17 16 14 12 9 10 9 18/30 Read block 424 len 32768 19/30 Flush 20/30 Write block 545 len 4096 data: 16 21/30 Read block 271 len 36864 22/30 Read block 148 len 12288 23/30 Read block 574 len 16384 24/30 Read block 419 len 24576 25/30 Write block 300 len 4096 data: 17 26/30 Read block 36 len 12288 27/30 Write block 306 len 32768 data: 15 16 19 22 21 17 14 14 28/30 Write block 514 len 36864 data: 15 15 20 19 19 21 20 18 14 29/30 Write block 199 len 16384 data: 16 17 15 17 30/30 Read block 452 len 8192 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 1 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 4 Sent Sent Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 9 9 11 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:16:59.078869585Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1939} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 187 187 187 337 337 337 F F F 2 040-059 186 186 186 335 335 335 F F F 3 060-079 183 183 183 330 330 330 F F F 4 080-099 188 188 188 339 339 339 F F F 5 100-119 188 188 188 340 340 340 F F F 6 120-139 188 188 188 340 340 340 F F F 7 140-159 183 183 183 330 330 330 F F F 8 160-179 184 184 184 332 332 332 F F F 9 180-199 188 188 188 340 340 340 T T T 10 200-219 188 188 188 339 339 339 T T T 11 220-239 188 188 188 339 339 339 F F F 12 240-259 187 187 187 337 337 337 F F F 13 260-279 188 188 188 339 339 339 F F F 14 280-299 186 186 186 335 335 335 F F F 15 300-319 186 186 186 336 336 336 T T T 16 320-339 186 186 186 336 336 336 F F F 17 340-359 184 184 184 331 331 331 F F F 18 360-379 184 184 184 331 331 331 F F F 19 380-399 186 186 186 335 335 335 F F F 20 400-419 184 184 184 331 331 331 F F F 21 420-439 184 184 184 332 332 332 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 188 188 188 340 340 340 F F F 25 500-519 187 187 187 337 337 337 T T T 26 520-539 178 178 178 321 321 321 T T T 27 540-559 188 188 188 340 340 340 T T T 28 560-579 188 188 188 339 339 339 F F F 29 580-599 186 186 186 335 335 335 F F F Max gen: 188, Max flush: 340 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 187 187 187 337 337 337 F F F 2 040-059 186 186 186 335 335 335 F F F 3 060-079 183 183 183 330 330 330 F F F 4 080-099 188 188 188 339 339 339 F F F 5 100-119 188 188 188 340 340 340 F F F 6 120-139 188 188 188 340 340 340 F F F 7 140-159 183 183 183 330 330 330 F F F 8 160-179 184 184 184 332 332 332 F F F 9 180-199 189 189 189 341 341 341 F F F 10 200-219 189 189 189 341 341 341 F F F 11 220-239 188 188 188 339 339 339 F F F 12 240-259 187 187 187 337 337 337 F F F 13 260-279 188 188 188 339 339 339 F F F 14 280-299 186 186 186 335 335 335 F F F 15 300-319 189 189 189 341 341 341 F F F 16 320-339 186 186 186 336 336 336 F F F 17 340-359 184 184 184 331 331 331 F F F 18 360-379 184 184 184 331 331 331 F F F 19 380-399 186 186 186 335 335 335 F F F 20 400-419 184 184 184 331 331 331 F F F 21 420-439 184 184 184 332 332 332 F F F 22 440-459 180 180 180 323 323 323 F F F 23 460-479 174 174 174 313 313 313 F F F 24 480-499 188 188 188 340 340 340 F F F 25 500-519 189 189 189 341 341 341 F F F 26 520-539 189 189 189 341 341 341 F F F 27 540-559 189 189 189 341 341 341 F F F 28 560-579 188 188 188 339 339 339 F F F 29 580-599 186 186 186 335 335 335 F F F Max gen: 189, Max flush: 341 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.318589422Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.361940671Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.362515365Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.362556001Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.362571099Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.362585348Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947} {"msg":"Crucible 1c2c5e50-e8ad-49b5-9e63-8f079bb1516d has session id: 5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363143703Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Upstairs opts: Upstairs UUID: 1c2c5e50-e8ad-49b5-9e63-8f079bb1516d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.36317734Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Crucible stats registered with UUID: 1c2c5e50-e8ad-49b5-9e63-8f079bb1516d","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363193889Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:17:03.363209947Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.3636902Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363726267Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947} {"msg":"1c2c5e50-e8ad-49b5-9e63-8f079bb1516d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363822077Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363843835Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"0","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363865483Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"0","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363879722Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"1","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363893561Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"1","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363906619Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"2","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363921198Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"2","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363933857Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363947175Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363960394Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"ds_connection connected from Ok(127.0.0.1:38775)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.363979702Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"ds_connection connected from Ok(127.0.0.1:59234)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.364065364Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"ds_connection connected from Ok(127.0.0.1:39882)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.364304981Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.364689013Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"2","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.36472608Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"0","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.36482733Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"1","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.365287555Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"0","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.36533367Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"1","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.365433611Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"2","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.365858329Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.365881937Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"[0]R flush_numbers[0..12]: [322, 337, 335, 330, 339, 340, 340, 330, 332, 341, 341, 339]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.365995066Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"[0]R generation[0..12]: [179, 187, 186, 183, 188, 188, 188, 183, 184, 189, 189, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366023653Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366039881Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"[1]R flush_numbers[0..12]: [322, 337, 335, 330, 339, 340, 340, 330, 332, 341, 341, 339]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.36605455Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"[1]R generation[0..12]: [179, 187, 186, 183, 188, 188, 188, 183, 184, 189, 189, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366068669Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366082717Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"[2]R flush_numbers[0..12]: [322, 337, 335, 330, 339, 340, 340, 330, 332, 341, 341, 339]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366099646Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"[2]R generation[0..12]: [179, 187, 186, 183, 188, 188, 188, 183, 184, 189, 189, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366113614Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-28T17:17:03.366127353Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366143291Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.36615648Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Next flush: 342","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366169029Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366181678Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"":"downstairs","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366194406Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366206075Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"0","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366218984Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"1","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366231623Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"client":"2","":"downstairs","session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366244681Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.36625677Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"1c2c5e50-e8ad-49b5-9e63-8f079bb1516d is now active with session: 5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366269139Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:03.366280818Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947,"session_id":"5c7a21fb-9b24-4d75-bb4b-7c621c6e3c77"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-28T17:17:03.366295177Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 441 len 32768 data: 15 17 18 19 20 19 20 18 02/30 Read block 181 len 24576 03/30 Write block 146 len 28672 data: 9 10 7 10 11 9 8 04/30 Write block 454 len 28672 data: 13 13 15 12 11 13 14 05/30 Write block 484 len 32768 data: 11 10 12 16 13 15 14 13 06/30 Write block 426 len 40960 data: 11 12 16 18 20 18 16 19 23 20 07/30 Read block 312 len 24576 08/30 Flush 09/30 Write block 370 len 16384 data: 12 13 12 10 10/30 Read block 407 len 8192 11/30 Read block 50 len 28672 12/30 Write block 277 len 24576 data: 17 17 17 13 11 11 13/30 Read block 462 len 20480 14/30 Write block 96 len 16384 data: 15 15 12 12 15/30 Read block 403 len 16384 16/30 Read block 210 len 8192 17/30 Write block 346 len 36864 data: 17 17 16 15 12 12 11 14 13 18/30 Write block 161 len 8192 data: 18 19 19/30 Write block 120 len 12288 data: 10 9 8 20/30 Read block 131 len 4096 21/30 Write block 74 len 40960 data: 16 15 13 12 13 14 14 15 14 15 22/30 Read block 74 len 36864 23/30 Write block 347 len 8192 data: 18 17 24/30 Write block 51 len 24576 data: 15 16 13 13 11 8 25/30 Write block 320 len 20480 data: 13 14 18 16 18 26/30 Read block 379 len 36864 27/30 Write block 571 len 20480 data: 16 16 16 14 11 28/30 Write block 176 len 8192 data: 18 17 29/30 Read block 39 len 24576 30/30 Read block 533 len 16384 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 4 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 22 21 22 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:17:03.38751959Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1947} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 187 187 187 337 337 337 F F F 2 040-059 186 186 186 335 335 335 T T T 3 060-079 183 183 183 330 330 330 T T T 4 080-099 188 188 188 339 339 339 T T T 5 100-119 188 188 188 340 340 340 F F F 6 120-139 188 188 188 340 340 340 T T T 7 140-159 190 190 190 342 342 342 F F F 8 160-179 184 184 184 332 332 332 T T T 9 180-199 189 189 189 341 341 341 F F F 10 200-219 189 189 189 341 341 341 F F F 11 220-239 188 188 188 339 339 339 F F F 12 240-259 187 187 187 337 337 337 F F F 13 260-279 188 188 188 339 339 339 T T T 14 280-299 186 186 186 335 335 335 T T T 15 300-319 189 189 189 341 341 341 F F F 16 320-339 186 186 186 336 336 336 T T T 17 340-359 184 184 184 331 331 331 T T T 18 360-379 184 184 184 331 331 331 T T T 19 380-399 186 186 186 335 335 335 F F F 20 400-419 184 184 184 331 331 331 F F F 21 420-439 190 190 190 342 342 342 F F F 22 440-459 190 190 190 342 342 342 F F F 23 460-479 190 190 190 342 342 342 F F F 24 480-499 190 190 190 342 342 342 F F F 25 500-519 189 189 189 341 341 341 F F F 26 520-539 189 189 189 341 341 341 F F F 27 540-559 189 189 189 341 341 341 F F F 28 560-579 188 188 188 339 339 339 T T T 29 580-599 186 186 186 335 335 335 F F F Max gen: 190, Max flush: 342 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 322 322 322 F F F 1 020-039 187 187 187 337 337 337 F F F 2 040-059 191 191 191 343 343 343 F F F 3 060-079 191 191 191 343 343 343 F F F 4 080-099 191 191 191 343 343 343 F F F 5 100-119 188 188 188 340 340 340 F F F 6 120-139 191 191 191 343 343 343 F F F 7 140-159 190 190 190 342 342 342 F F F 8 160-179 191 191 191 343 343 343 F F F 9 180-199 189 189 189 341 341 341 F F F 10 200-219 189 189 189 341 341 341 F F F 11 220-239 188 188 188 339 339 339 F F F 12 240-259 187 187 187 337 337 337 F F F 13 260-279 191 191 191 343 343 343 F F F 14 280-299 191 191 191 343 343 343 F F F 15 300-319 189 189 189 341 341 341 F F F 16 320-339 191 191 191 343 343 343 F F F 17 340-359 191 191 191 343 343 343 F F F 18 360-379 191 191 191 343 343 343 F F F 19 380-399 186 186 186 335 335 335 F F F 20 400-419 184 184 184 331 331 331 F F F 21 420-439 190 190 190 342 342 342 F F F 22 440-459 190 190 190 342 342 342 F F F 23 460-479 190 190 190 342 342 342 F F F 24 480-499 190 190 190 342 342 342 F F F 25 500-519 189 189 189 341 341 341 F F F 26 520-539 189 189 189 341 341 341 F F F 27 540-559 189 189 189 341 341 341 F F F 28 560-579 191 191 191 343 343 343 F F F 29 580-599 186 186 186 335 335 335 F F F Max gen: 191, Max flush: 343 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.66151119Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:17:07.703958407Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.704349769Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.735752221Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.736325045Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.736366931Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.73638347Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.736401298Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955} {"msg":"Crucible 1a9fb469-b367-4503-9e9f-124942a6a3d2 has session id: 2579ed42-2f06-4c13-b107-0af8bca63fdc","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.736988581Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Upstairs opts: Upstairs UUID: 1a9fb469-b367-4503-9e9f-124942a6a3d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737034816Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Crucible stats registered with UUID: 1a9fb469-b367-4503-9e9f-124942a6a3d2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737053584Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:17:07.737068703Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737432847Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737468504Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955} {"msg":"1a9fb469-b367-4503-9e9f-124942a6a3d2 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737499431Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737514439Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"0","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737534948Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"0","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737552736Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"1","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737567384Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"1","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737580243Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"2","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737594062Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"2","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737606761Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737619999Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737632988Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"ds_connection connected from Ok(127.0.0.1:59163)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737655216Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"ds_connection connected from Ok(127.0.0.1:53388)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737746377Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"ds_connection connected from Ok(127.0.0.1:49421)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.737783713Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.738377375Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"1","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.738406213Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"0","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.738421161Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"2","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.738983016Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"1","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739014113Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"0","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.73915147Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"2","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739639113Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739660421Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"[0]R flush_numbers[0..12]: [322, 337, 343, 343, 343, 340, 343, 342, 343, 341, 341, 339]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739699947Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"[0]R generation[0..12]: [179, 187, 191, 191, 191, 188, 191, 190, 191, 189, 189, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739717255Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739731174Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"[1]R flush_numbers[0..12]: [322, 337, 343, 343, 343, 340, 343, 342, 343, 341, 341, 339]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739744612Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"[1]R generation[0..12]: [179, 187, 191, 191, 191, 188, 191, 190, 191, 189, 189, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739758091Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.73977166Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"[2]R flush_numbers[0..12]: [322, 337, 343, 343, 343, 340, 343, 342, 343, 341, 341, 339]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739785109Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"[2]R generation[0..12]: [179, 187, 191, 191, 191, 188, 191, 190, 191, 189, 189, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739798617Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739812446Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739824814Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739840763Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Next flush: 344","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739859801Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.7398725Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"":"downstairs","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739885679Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739897658Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"0","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739910906Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"1","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739923865Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"client":"2","":"downstairs","session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739936914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739948762Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"1a9fb469-b367-4503-9e9f-124942a6a3d2 is now active with session: 2579ed42-2f06-4c13-b107-0af8bca63fdc","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739961031Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.73997259Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955,"session_id":"2579ed42-2f06-4c13-b107-0af8bca63fdc"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:07.739984439Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 10 len 20480 data: 16 16 16 17 17 02/30 Write block 179 len 16384 data: 18 19 16 17 03/30 Flush 04/30 Write block 563 len 28672 data: 9 9 8 9 10 13 14 05/30 Read block 160 len 20480 06/30 Write block 6 len 32768 data: 10 13 13 15 17 17 17 18 07/30 Read block 135 len 28672 08/30 Read block 453 len 36864 09/30 Read block 140 len 40960 10/30 Write block 332 len 28672 data: 13 16 18 17 17 19 17 11/30 Read block 396 len 24576 12/30 Read block 118 len 40960 13/30 Read block 449 len 4096 14/30 Read block 13 len 24576 15/30 Read block 174 len 12288 16/30 Write block 195 len 40960 data: 15 16 16 17 17 18 16 18 18 18 17/30 Flush 18/30 Write block 81 len 24576 data: 16 15 16 15 14 13 19/30 Read block 322 len 28672 20/30 Read block 501 len 4096 21/30 Write block 400 len 24576 data: 19 21 18 22 18 18 22/30 Read block 485 len 40960 23/30 Read block 417 len 20480 24/30 Read block 172 len 16384 25/30 Flush 26/30 Write block 299 len 24576 data: 14 18 18 15 16 13 27/30 Write block 259 len 8192 data: 13 12 28/30 Read block 418 len 4096 29/30 Read block 210 len 28672 30/30 Write block 542 len 4096 data: 14 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:17:07.760673695Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1955} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 345 345 345 F F F 1 020-039 187 187 187 337 337 337 F F F 2 040-059 191 191 191 343 343 343 F F F 3 060-079 191 191 191 343 343 343 F F F 4 080-099 192 192 192 346 346 346 F F F 5 100-119 188 188 188 340 340 340 F F F 6 120-139 191 191 191 343 343 343 F F F 7 140-159 190 190 190 342 342 342 F F F 8 160-179 192 192 192 344 344 344 F F F 9 180-199 192 192 192 345 345 345 F F F 10 200-219 192 192 192 345 345 345 F F F 11 220-239 188 188 188 339 339 339 F F F 12 240-259 187 187 187 337 337 337 T T T 13 260-279 191 191 191 343 343 343 T T T 14 280-299 191 191 191 343 343 343 T T T 15 300-319 189 189 189 341 341 341 T T T 16 320-339 192 192 192 345 345 345 F F F 17 340-359 191 191 191 343 343 343 F F F 18 360-379 191 191 191 343 343 343 F F F 19 380-399 186 186 186 335 335 335 F F F 20 400-419 192 192 192 346 346 346 F F F 21 420-439 190 190 190 342 342 342 F F F 22 440-459 190 190 190 342 342 342 F F F 23 460-479 190 190 190 342 342 342 F F F 24 480-499 190 190 190 342 342 342 F F F 25 500-519 189 189 189 341 341 341 F F F 26 520-539 189 189 189 341 341 341 F F F 27 540-559 189 189 189 341 341 341 T T T 28 560-579 192 192 192 345 345 345 F F F 29 580-599 186 186 186 335 335 335 F F F Max gen: 192, Max flush: 346 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 345 345 345 F F F 1 020-039 187 187 187 337 337 337 F F F 2 040-059 191 191 191 343 343 343 F F F 3 060-079 191 191 191 343 343 343 F F F 4 080-099 192 192 192 346 346 346 F F F 5 100-119 188 188 188 340 340 340 F F F 6 120-139 191 191 191 343 343 343 F F F 7 140-159 190 190 190 342 342 342 F F F 8 160-179 192 192 192 344 344 344 F F F 9 180-199 192 192 192 345 345 345 F F F 10 200-219 192 192 192 345 345 345 F F F 11 220-239 188 188 188 339 339 339 F F F 12 240-259 193 193 193 347 347 347 F F F 13 260-279 193 193 193 347 347 347 F F F 14 280-299 193 193 193 347 347 347 F F F 15 300-319 193 193 193 347 347 347 F F F 16 320-339 192 192 192 345 345 345 F F F 17 340-359 191 191 191 343 343 343 F F F 18 360-379 191 191 191 343 343 343 F F F 19 380-399 186 186 186 335 335 335 F F F 20 400-419 192 192 192 346 346 346 F F F 21 420-439 190 190 190 342 342 342 F F F 22 440-459 190 190 190 342 342 342 F F F 23 460-479 190 190 190 342 342 342 F F F 24 480-499 190 190 190 342 342 342 F F F 25 500-519 189 189 189 341 341 341 F F F 26 520-539 189 189 189 341 341 341 F F F 27 540-559 193 193 193 347 347 347 F F F 28 560-579 192 192 192 345 345 345 F F F 29 580-599 186 186 186 335 335 335 F F F Max gen: 193, Max flush: 347 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.024819351Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:17:12.066650568Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.06703576Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.097974738Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.098539033Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.098581009Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.098599587Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.098617165Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963} {"msg":"Crucible e4087022-57a2-4530-aa54-7529f5a996d9 has session id: e8713ee1-e06e-4227-95f6-c23018a55897","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.099220247Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Upstairs opts: Upstairs UUID: e4087022-57a2-4530-aa54-7529f5a996d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.099265242Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Crucible stats registered with UUID: e4087022-57a2-4530-aa54-7529f5a996d9","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.09928986Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:17:12.099306768Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.099729287Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.099762974Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963} {"msg":"e4087022-57a2-4530-aa54-7529f5a996d9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.099816488Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.099849165Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"0","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.099887662Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"0","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.099918709Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"1","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.099948066Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"1","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.099977153Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"2","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.100002761Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"2","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.100024958Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.100045786Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.100067034Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"ds_connection connected from Ok(127.0.0.1:39458)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.100120609Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"ds_connection connected from Ok(127.0.0.1:37440)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.1002127Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"ds_connection connected from Ok(127.0.0.1:59857)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.100245097Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.100768706Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"2","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.100809342Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"0","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.100885375Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"1","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.101397814Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"2","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.10144326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"1","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.101578027Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"0","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102035572Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.10206111Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"[0]R flush_numbers[0..12]: [345, 337, 343, 343, 346, 340, 343, 342, 344, 345, 345, 339]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102088777Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"[0]R generation[0..12]: [192, 187, 191, 191, 192, 188, 191, 190, 192, 192, 192, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102104266Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102118324Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"[1]R flush_numbers[0..12]: [345, 337, 343, 343, 346, 340, 343, 342, 344, 345, 345, 339]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102132073Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"[1]R generation[0..12]: [192, 187, 191, 191, 192, 188, 191, 190, 192, 192, 192, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102150201Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.10216427Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"[2]R flush_numbers[0..12]: [345, 337, 343, 343, 346, 340, 343, 342, 344, 345, 345, 339]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102177729Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"[2]R generation[0..12]: [192, 187, 191, 191, 192, 188, 191, 190, 192, 192, 192, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102191117Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102204696Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102217475Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102230324Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Next flush: 348","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102242902Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"}Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.10227023Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"":"downstairs","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102294737Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102309976Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"0","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102325914Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"1","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102348802Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"client":"2","":"downstairs","session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.10236504Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102377329Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"e4087022-57a2-4530-aa54-7529f5a996d9 is now active with session: e8713ee1-e06e-4227-95f6-c23018a55897","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102403807Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102422715Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963,"session_id":"e8713ee1-e06e-4227-95f6-c23018a55897"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:12.102435384Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 368 len 40960 data: 11 12 13 14 13 11 11 11 13 12 02/30 Write block 171 len 20480 data: 16 19 20 19 19 03/30 Read block 102 len 16384 04/30 Read block 595 len 20480 05/30 Write block 580 len 8192 data: 17 17 06/30 Write block 594 len 16384 data: 16 12 11 8 07/30 Read block 241 len 8192 08/30 Read block 434 len 12288 09/30 Read block 405 len 32768 10/30 Read block 149 len 8192 11/30 Read block 330 len 32768 12/30 Read block 146 len 16384 13/30 Read block 302 len 20480 14/30 Read block 512 len 28672 15/30 Read block 262 len 28672 16/30 Read block 584 len 36864 17/30 Write block 105 len 12288 data: 11 11 15 18/30 Read block 497 len 32768 19/30 Write block 500 len 24576 data: 19 16 16 20 19 18 20/30 Read block 289 len 36864 21/30 Read block 560 len 24576 22/30 Read block 94 len 36864 23/30 Write block 297 len 40960 data: 14 16 15 19 19 16 17 14 13 16 24/30 Read block 550 len 20480 25/30 Write block 513 len 32768 data: 18 16 16 21 20 20 22 21 26/30 Write block 588 len 36864 data: 18 22 19 19 19 18 17 13 12 27/30 Read block 416 len 24576 28/30 Read block 125 len 8192 29/30 Write block 437 len 28672 data: 19 19 16 15 16 18 19 30/30 Write block 439 len 28672 data: 17 16 17 19 20 20 21 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:17:12.122788032Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1963} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 345 345 345 F F F 1 020-039 187 187 187 337 337 337 F F F 2 040-059 191 191 191 343 343 343 F F F 3 060-079 191 191 191 343 343 343 F F F 4 080-099 192 192 192 346 346 346 F F F 5 100-119 188 188 188 340 340 340 T T T 6 120-139 191 191 191 343 343 343 F F F 7 140-159 190 190 190 342 342 342 F F F 8 160-179 192 192 192 344 344 344 T T T 9 180-199 192 192 192 345 345 345 F F F 10 200-219 192 192 192 345 345 345 F F F 11 220-239 188 188 188 339 339 339 F F F 12 240-259 193 193 193 347 347 347 F F F 13 260-279 193 193 193 347 347 347 F F F 14 280-299 193 193 193 347 347 347 T T T 15 300-319 193 193 193 347 347 347 T T T 16 320-339 192 192 192 345 345 345 F F F 17 340-359 191 191 191 343 343 343 F F F 18 360-379 191 191 191 343 343 343 T T T 19 380-399 186 186 186 335 335 335 F F F 20 400-419 192 192 192 346 346 346 F F F 21 420-439 190 190 190 342 342 342 T T T 22 440-459 190 190 190 342 342 342 T T T 23 460-479 190 190 190 342 342 342 F F F 24 480-499 190 190 190 342 342 342 F F F 25 500-519 189 189 189 341 341 341 T T T 26 520-539 189 189 189 341 341 341 T T T 27 540-559 193 193 193 347 347 347 F F F 28 560-579 192 192 192 345 345 345 F F F 29 580-599 186 186 186 335 335 335 T T T Max gen: 193, Max flush: 347 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 345 345 345 F F F 1 020-039 187 187 187 337 337 337 F F F 2 040-059 191 191 191 343 343 343 F F F 3 060-079 191 191 191 343 343 343 F F F 4 080-099 192 192 192 346 346 346 F F F 5 100-119 194 194 194 348 348 348 F F F 6 120-139 191 191 191 343 343 343 F F F 7 140-159 190 190 190 342 342 342 F F F 8 160-179 194 194 194 348 348 348 F F F 9 180-199 192 192 192 345 345 345 F F F 10 200-219 192 192 192 345 345 345 F F F 11 220-239 188 188 188 339 339 339 F F F 12 240-259 193 193 193 347 347 347 F F F 13 260-279 193 193 193 347 347 347 F F F 14 280-299 194 194 194 348 348 348 F F F 15 300-319 194 194 194 348 348 348 F F F 16 320-339 192 192 192 345 345 345 F F F 17 340-359 191 191 191 343 343 343 F F F 18 360-379 194 194 194 348 348 348 F F F 19 380-399 186 186 186 335 335 335 F F F 20 400-419 192 192 192 346 346 346 F F F 21 420-439 194 194 194 348 348 348 F F F 22 440-459 194 194 194 348 348 348 F F F 23 460-479 190 190 190 342 342 342 F F F 24 480-499 190 190 190 342 342 342 F F F 25 500-519 194 194 194 348 348 348 F F F 26 520-539 194 194 194 348 348 348 F F F 27 540-559 193 193 193 347 347 347 F F F 28 560-579 192 192 192 345 345 345 F F F 29 580-599 194 194 194 348 348 348 F F F Max gen: 194, Max flush: 348 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.419229825Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.464147961Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.464760361Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.464799287Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.464814416Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.464829554Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971} {"msg":"Crucible 2b5d67ea-8068-4fdc-9b89-62066f63ec05 has session id: 40b65002-6e2c-41b7-887e-a00f2ca5133f","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465337555Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Upstairs opts: Upstairs UUID: 2b5d67ea-8068-4fdc-9b89-62066f63ec05, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.46538563Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Crucible stats registered with UUID: 2b5d67ea-8068-4fdc-9b89-62066f63ec05","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465413388Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:17:16.465434135Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465836056Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465860424Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971} {"msg":"2b5d67ea-8068-4fdc-9b89-62066f63ec05 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465872973Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465885531Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"0","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465910619Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"0","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465926018Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"1","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465940186Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"1","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465953265Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"2","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465966964Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"2","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465979862Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.465993231Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.46600652Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62683)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.466162844Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61691)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.466392682Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40291)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.466433198Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.46683081Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"2","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.467019371Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"0","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.467421092Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"2","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.467537731Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"1","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.467566348Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"0","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.467989127Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468117494Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468230253Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"1","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"[0]R flush_numbers[0..12]: [345, 337, 343, 343, 346, 348, 343, 342, 348, 345, 345, 339]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468696448Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"[0]R generation[0..12]: [192, 187, 191, 191, 192, 194, 191, 190, 194, 192, 192, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468716836Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468731674Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"[1]R flush_numbers[0..12]: [345, 337, 343, 343, 346, 348, 343, 342, 348, 345, 345, 339]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468745843Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"[1]R generation[0..12]: [192, 187, 191, 191, 192, 194, 191, 190, 194, 192, 192, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468759252Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468772801Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"[2]R flush_numbers[0..12]: [345, 337, 343, 343, 346, 348, 343, 342, 348, 345, 345, 339]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468786369Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"[2]R generation[0..12]: [192, 187, 191, 191, 192, 194, 191, 190, 194, 192, 192, 188]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468799778Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468813537Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Max found gen is 195","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468830625Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Generation requested: 196 >= found:195","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468843954Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468867641Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468891189Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"":"downstairs","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468916716Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468939324Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"0","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468965762Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"1","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.468992669Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"client":"2","":"downstairs","session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.469040564Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.469067062Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"2b5d67ea-8068-4fdc-9b89-62066f63ec05 is now active with session: 40b65002-6e2c-41b7-887e-a00f2ca5133f","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-28T17:17:16.469092759Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.469116567Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971,"session_id":"40b65002-6e2c-41b7-887e-a00f2ca5133f"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:16.469139935Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 30 len 28672 data: 17 16 15 16 18 15 14 02/30 Read block 479 len 32768 03/30 Write block 340 len 16384 data: 14 14 15 15 04/30 Flush 05/30 Write block 265 len 20480 data: 20 21 21 17 15 06/30 Write block 71 len 20480 data: 16 18 17 17 16 07/30 Write block 87 len 12288 data: 16 18 19 08/30 Flush 09/30 Write block 33 len 40960 data: 17 19 16 15 15 18 14 13 12 12 10/30 Read block 448 len 28672 11/30 Read block 327 len 16384 12/30 Write block 229 len 20480 data: 17 13 11 9 9 13/30 Read block 130 len 40960 14/30 Flush 15/30 Write block 108 len 20480 data: 15 16 16 13 12 16/30 Write block 71 len 20480 data: 17 19 18 18 17 17/30 Read block 80 len 12288 18/30 Write block 481 len 28672 data: 14 14 13 12 11 13 17 19/30 Read block 252 len 8192 20/30 Write block 439 len 12288 data: 18 17 18 21/30 Read block 587 len 32768 22/30 Flush 23/30 Write block 131 len 40960 data: 15 15 15 13 14 12 16 19 21 25 24/30 Read block 319 len 36864 25/30 Write block 237 len 8192 data: 19 20 26/30 Write block 113 len 24576 data: 16 18 17 15 13 10 27/30 Write block 376 len 12288 data: 14 13 13 28/30 Read block 270 len 20480 29/30 Read block 184 len 4096 30/30 Write block 258 len 12288 data: 12 14 13 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:17:16.488273522Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1971} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 345 345 345 F F F 1 020-039 196 196 196 351 351 351 F F F 2 040-059 196 196 196 351 351 351 F F F 3 060-079 196 196 196 352 352 352 F F F 4 080-099 196 196 196 350 350 350 F F F 5 100-119 196 196 196 352 352 352 T T T 6 120-139 191 191 191 343 343 343 T T T 7 140-159 190 190 190 342 342 342 T T T 8 160-179 194 194 194 348 348 348 F F F 9 180-199 192 192 192 345 345 345 F F F 10 200-219 192 192 192 345 345 345 F F F 11 220-239 196 196 196 351 351 351 T T T 12 240-259 193 193 193 347 347 347 T T T 13 260-279 196 196 196 350 350 350 T T T 14 280-299 194 194 194 348 348 348 F F F 15 300-319 194 194 194 348 348 348 F F F 16 320-339 192 192 192 345 345 345 F F F 17 340-359 196 196 196 349 349 349 F F F 18 360-379 194 194 194 348 348 348 T T T 19 380-399 186 186 186 335 335 335 F F F 20 400-419 192 192 192 346 346 346 F F F 21 420-439 196 196 196 352 352 352 F F F 22 440-459 196 196 196 352 352 352 F F F 23 460-479 190 190 190 342 342 342 F F F 24 480-499 196 196 196 352 352 352 F F F 25 500-519 194 194 194 348 348 348 F F F 26 520-539 194 194 194 348 348 348 F F F 27 540-559 193 193 193 347 347 347 F F F 28 560-579 192 192 192 345 345 345 F F F 29 580-599 194 194 194 348 348 348 F F F Max gen: 196, Max flush: 352 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 345 345 345 F F F 1 020-039 196 196 196 351 351 351 F F F 2 040-059 196 196 196 351 351 351 F F F 3 060-079 196 196 196 352 352 352 F F F 4 080-099 196 196 196 350 350 350 F F F 5 100-119 197 197 197 353 353 353 F F F 6 120-139 197 197 197 353 353 353 F F F 7 140-159 197 197 197 353 353 353 F F F 8 160-179 194 194 194 348 348 348 F F F 9 180-199 192 192 192 345 345 345 F F F 10 200-219 192 192 192 345 345 345 F F F 11 220-239 197 197 197 353 353 353 F F F 12 240-259 197 197 197 353 353 353 F F F 13 260-279 197 197 197 353 353 353 F F F 14 280-299 194 194 194 348 348 348 F F F 15 300-319 194 194 194 348 348 348 F F F 16 320-339 192 192 192 345 345 345 F F F 17 340-359 196 196 196 349 349 349 F F F 18 360-379 197 197 197 353 353 353 F F F 19 380-399 186 186 186 335 335 335 F F F 20 400-419 192 192 192 346 346 346 F F F 21 420-439 196 196 196 352 352 352 F F F 22 440-459 196 196 196 352 352 352 F F F 23 460-479 190 190 190 342 342 342 F F F 24 480-499 196 196 196 352 352 352 F F F 25 500-519 194 194 194 348 348 348 F F F 26 520-539 194 194 194 348 348 348 F F F 27 540-559 193 193 193 347 347 347 F F F 28 560-579 192 192 192 345 345 345 F F F 29 580-599 194 194 194 348 348 348 F F F Max gen: 197, Max flush: 353 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.736477712Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.779517711Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.780104444Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"752fd3e208a28c201ec7c5e297caafe64bf1aaf2\",\n git_commit_timestamp: \"2025-01-28T16:52:48.000000000Z\",\n git_branch: \"alan/where-my-writes-at\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.78014603Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.780162918Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.780177387Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979} {"msg":"Crucible a4e0b00c-612b-46d0-84ae-a83e00c2fa7a has session id: cfca093c-a87b-4d0c-8452-7249410c58e2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.780703025Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"Upstairs opts: Upstairs UUID: a4e0b00c-612b-46d0-84ae-a83e00c2fa7a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.780749461Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"Crucible stats registered with UUID: a4e0b00c-612b-46d0-84ae-a83e00c2fa7a","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.780777428Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:17:20.780797996Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781303917Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781328694Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979} {"msg":"a4e0b00c-612b-46d0-84ae-a83e00c2fa7a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781385459Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781411776Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"0","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781436394Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"0","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781451283Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"1","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781468201Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"1","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781492408Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"2","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781518736Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"2","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781543554Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781569311Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781591309Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43245)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781618436Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:35218)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781650863Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:36129)","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.781675751Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.782240436Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"2","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.782276372Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"0","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.782885443Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"0","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.78291976Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"2","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.783380435Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"1","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.783567326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.783601413Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.783889815Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"1","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"[0]R flush_numbers[0..12]: [345, 351, 351, 352, 350, 353, 353, 353, 348, 345, 345, 353]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784369818Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"[0]R generation[0..12]: [192, 196, 196, 196, 196, 197, 197, 197, 194, 192, 192, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784390326Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784404945Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"[1]R flush_numbers[0..12]: [345, 351, 351, 352, 350, 353, 353, 353, 348, 345, 345, 353]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784418774Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"[1]R generation[0..12]: [192, 196, 196, 196, 196, 197, 197, 197, 194, 192, 192, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784435982Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784450171Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"[2]R flush_numbers[0..12]: [345, 351, 351, 352, 350, 353, 353, 353, 348, 345, 345, 353]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784463859Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"[2]R generation[0..12]: [192, 196, 196, 196, 196, 197, 197, 197, 194, 192, 192, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784477168Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784491467Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784504375Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784523243Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"Next flush: 354","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784536122Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784549961Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"":"downstairs","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.78456268Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784574628Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"0","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784587957Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"1","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784601116Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"client":"2","":"downstairs","session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784614295Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784626453Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"a4e0b00c-612b-46d0-84ae-a83e00c2fa7a is now active with session: cfca093c-a87b-4d0c-8452-7249410c58e2","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784641172Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.784653051Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979,"session_id":"cfca093c-a87b-4d0c-8452-7249410c58e2"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-01-28T17:17:20.78466491Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 500 len 40960 data: 20 17 17 21 20 19 17 12 12 15 02/30 Read block 276 len 28672 03/30 Write block 524 len 24576 data: 14 13 9 8 8 12 04/30 Read block 494 len 4096 05/30 Read block 243 len 40960 06/30 Read block 414 len 24576 07/30 Write block 162 len 20480 data: 20 21 26 19 16 08/30 Read block 83 len 12288 09/30 Read block 456 len 24576 10/30 Read block 587 len 20480 11/30 Read block 203 len 8192 12/30 Read block 292 len 32768 13/30 Write block 524 len 32768 data: 15 14 10 9 9 13 16 19 14/30 Write block 282 len 12288 data: 12 10 8 15/30 Write block 152 len 24576 data: 9 9 10 10 10 10 16/30 Read block 22 len 28672 17/30 Write block 239 len 32768 data: 20 21 21 23 18 18 17 15 18/30 Write block 253 len 40960 data: 15 16 13 14 12 13 15 14 16 19 19/30 Write block 568 len 4096 data: 14 20/30 Read block 385 len 16384 21/30 Read block 529 len 36864 22/30 Read block 115 len 4096 23/30 Flush 24/30 Flush 25/30 Write block 67 len 12288 data: 17 15 17 26/30 Flush 27/30 Write block 447 len 20480 data: 21 19 15 18 16 28/30 Write block 281 len 32768 data: 12 13 11 9 12 10 10 10 29/30 Write block 418 len 36864 data: 11 12 14 14 13 13 10 11 12 30/30 Read block 37 len 20480 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 3 4 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-28T17:17:20.805136126Z","hostname":"w-01JJPY45MXXEQWS7EPC9XWQB28","pid":1979} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 345 345 345 F F F 1 020-039 196 196 196 351 351 351 F F F 2 040-059 196 196 196 351 351 351 F F F 3 060-079 198 198 198 356 356 356 F F F 4 080-099 196 196 196 350 350 350 F F F 5 100-119 197 197 197 353 353 353 F F F 6 120-139 197 197 197 353 353 353 F F F 7 140-159 198 198 198 354 354 354 F F F 8 160-179 198 198 198 354 354 354 F F F 9 180-199 192 192 192 345 345 345 F F F 10 200-219 192 192 192 345 345 345 F F F 11 220-239 198 198 198 354 354 354 F F F 12 240-259 198 198 198 354 354 354 F F F 13 260-279 198 198 198 354 354 354 F F F 14 280-299 198 198 198 354 354 354 T T T 15 300-319 194 194 194 348 348 348 F F F 16 320-339 192 192 192 345 345 345 F F F 17 340-359 196 196 196 349 349 349 F F F 18 360-379 197 197 197 353 353 353 F F F 19 380-399 186 186 186 335 335 335 F F F 20 400-419 192 192 192 346 346 346 T T T 21 420-439 196 196 196 352 352 352 T T T 22 440-459 196 196 196 352 352 352 T T T 23 460-479 190 190 190 342 342 342 F F F 24 480-499 196 196 196 352 352 352 F F F 25 500-519 198 198 198 354 354 354 F F F 26 520-539 198 198 198 354 354 354 F F F 27 540-559 193 193 193 347 347 347 F F F 28 560-579 198 198 198 354 354 354 F F F 29 580-599 194 194 194 348 348 348 F F F Max gen: 198, Max flush: 356 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 345 345 345 F F F 1 020-039 196 196 196 351 351 351 F F F 2 040-059 196 196 196 351 351 351 F F F 3 060-079 198 198 198 356 356 356 F F F 4 080-099 196 196 196 350 350 350 F F F 5 100-119 197 197 197 353 353 353 F F F 6 120-139 197 197 197 353 353 353 F F F 7 140-159 198 198 198 354 354 354 F F F 8 160-179 198 198 198 354 354 354 F F F 9 180-199 192 192 192 345 345 345 F F F 10 200-219 192 192 192 345 345 345 F F F 11 220-239 198 198 198 354 354 354 F F F 12 240-259 198 198 198 354 354 354 F F F 13 260-279 198 198 198 354 354 354 F F F 14 280-299 199 199 199 357 357 357 F F F 15 300-319 194 194 194 348 348 348 F F F 16 320-339 192 192 192 345 345 345 F F F 17 340-359 196 196 196 349 349 349 F F F 18 360-379 197 197 197 353 353 353 F F F 19 380-399 186 186 186 335 335 335 F F F 20 400-419 199 199 199 357 357 357 F F F 21 420-439 199 199 199 357 357 357 F F F 22 440-459 199 199 199 357 357 357 F F F 23 460-479 190 190 190 342 342 342 F F F 24 480-499 196 196 196 352 352 352 F F F 25 500-519 198 198 198 354 354 354 F F F 26 520-539 198 198 198 354 354 354 F F F 27 540-559 193 193 193 347 347 347 F F F 28 560-579 198 198 198 354 354 354 F F F 29 580-599 194 194 194 348 348 348 F F F Max gen: 199, Max flush: 357 8:24 Test duration Test completed real 8:23.522488611 user 48.527924273 sys 18.292594820 trap 0.068912909 tflt 0.082039974 dflt 0.130043377 kflt 0.006959675 lock 13:23:59.198034140 slp 40:50.388314482 lat 5.655944773 stop 0.060400015