Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.27638307 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.021379504 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.020635724 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1190 1191 1192 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.718229579Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.798767646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.800210188Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.800369122Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.800389416Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.80040555Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194} {"msg":"Crucible 856cdf5e-cbab-4a6f-95e7-ee4da4f41e59 has session id: f5ffce96-15ff-4ffb-8dd7-b21713fa4f38","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.801245461Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Upstairs opts: Upstairs UUID: 856cdf5e-cbab-4a6f-95e7-ee4da4f41e59, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.801279038Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Crucible stats registered with UUID: 856cdf5e-cbab-4a6f-95e7-ee4da4f41e59","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.801294691Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:18:41.80133484Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802245677Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802325794Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194} {"msg":"856cdf5e-cbab-4a6f-95e7-ee4da4f41e59 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802356821Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802377775Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"0","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802506883Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"0","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.80253805Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"1","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802569847Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"1","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802597023Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"2","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802625589Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"2","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802651734Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.80267823Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802705256Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"ds_connection connected from Ok(127.0.0.1:53158)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802733652Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"io task","client":"2","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"ds_connection connected from Ok(127.0.0.1:45816)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802759247Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"io task","client":"1","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"ds_connection connected from Ok(127.0.0.1:61252)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.802786213Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.804233736Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"0","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.804274194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"1","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.804307872Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"2","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.805481865Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"0","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.805561292Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"1","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.805791092Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"2","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.806669381Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.806702989Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.80685131Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.806873265Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.806888228Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.806901751Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.806914914Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.806928607Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.80694201Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.806955293Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.806972336Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.806985239Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.807021037Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.80703498Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.807047363Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"downstairs","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.807060026Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.807071618Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"0","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.807084291Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"1","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.807096894Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"client":"2","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.807117258Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.807139643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"856cdf5e-cbab-4a6f-95e7-ee4da4f41e59 is now active with session: f5ffce96-15ff-4ffb-8dd7-b21713fa4f38","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:18:41.807160367Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.807180301Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.807202356Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194} {"msg":"Activated sub_volume 856cdf5e-cbab-4a6f-95e7-ee4da4f41e59","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:41.80722077Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:18:45.897532087Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.89763846Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:18:45.897675418Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:18:45.897702734Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.89772833Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1194,"":"io task","client":"0","":"downstairs","session_id":"f5ffce96-15ff-4ffb-8dd7-b21713fa4f38"} Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.929443743Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.979588773Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.980826329Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.980956758Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.980996656Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.981024252Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196} {"msg":"Crucible 78f09240-e9a3-4dad-94e5-8fa70959e953 has session id: faf231be-c844-4109-801d-e442b6cf3c50","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.981325367Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Upstairs opts: Upstairs UUID: 78f09240-e9a3-4dad-94e5-8fa70959e953, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.981367806Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Crucible stats registered with UUID: 78f09240-e9a3-4dad-94e5-8fa70959e953","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.981391531Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:18:45.981409726Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.981828346Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.981852161Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196} {"msg":"78f09240-e9a3-4dad-94e5-8fa70959e953 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.981929338Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.981947331Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"0","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.981974837Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"0","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.98198962Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"1","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.982003944Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"1","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.982016676Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"2","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.982030209Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"2","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.982042822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.982062026Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.982091282Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"ds_connection connected from Ok(127.0.0.1:54911)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.982208518Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"ds_connection connected from Ok(127.0.0.1:56939)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.982243926Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"ds_connection connected from Ok(127.0.0.1:56040)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.982273032Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.983056681Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"0","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.983095379Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"1","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.983791049Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"1","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.983824956Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"0","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.983891631Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"2","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.984430197Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.984464334Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.98448854Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"2","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985027996Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985056592Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985075657Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.98509048Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985104032Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985117995Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985131209Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985144241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985157694Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985170357Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985183Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-17T22:18:45.985202434Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985222718Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"":"downstairs","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985235841Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985247383Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"0","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985260126Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"1","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985272659Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"client":"2","":"downstairs","session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985288092Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985300065Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"78f09240-e9a3-4dad-94e5-8fa70959e953 is now active with session: faf231be-c844-4109-801d-e442b6cf3c50","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985312497Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.9853239Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196,"session_id":"faf231be-c844-4109-801d-e442b6cf3c50"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985335622Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196} {"msg":"Activated sub_volume 78f09240-e9a3-4dad-94e5-8fa70959e953","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:45.985346415Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 561 len 16384 data: 2 2 2 2 02/30 Read block 565 len 16384 03/30 Write block 558 len 40960 data: 2 2 2 3 3 3 3 2 2 2 04/30 Read block 557 len 4096 05/30 Write block 383 len 24576 data: 2 2 2 2 2 2 06/30 Read block 153 len 40960 07/30 Flush 08/30 Write block 566 len 40960 data: 3 3 2 2 2 2 2 2 2 2 09/30 Read block 11 len 28672 10/30 Write block 584 len 8192 data: 2 2 11/30 Write block 272 len 36864 data: 2 2 2 2 2 2 2 2 2 12/30 Flush 13/30 Write block 163 len 16384 data: 2 2 2 2 14/30 Read block 268 len 12288 15/30 Write block 428 len 12288 data: 2 2 2 16/30 Write block 441 len 24576 data: 2 2 2 2 2 2 17/30 Write block 234 len 20480 data: 2 2 2 2 2 18/30 Read block 194 len 8192 19/30 Read block 216 len 16384 20/30 Read block 29 len 28672 21/30 Read block 494 len 8192 22/30 Read block 122 len 12288 23/30 Read block 592 len 16384 24/30 Read block 463 len 36864 25/30 Read block 298 len 28672 26/30 Write block 38 len 28672 data: 2 2 2 2 2 2 2 27/30 Write block 461 len 20480 data: 2 2 2 2 2 28/30 Read block 407 len 4096 29/30 Write block 331 len 20480 data: 2 2 2 2 2 30/30 Write block 96 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 4 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:18:46.006599511Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1196} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 T T T 2 040-059 1 1 1 1 1 1 T T T 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 T T T 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 T T T 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 T T T 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 3 3 3 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 T T T 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 T T T 22 440-459 1 1 1 1 1 1 T T T 23 460-479 1 1 1 1 1 1 T T T 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 2 2 2 3 3 3 F F F 29 580-599 2 2 2 3 3 3 F F F Max gen: 2, Max flush: 3 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 3 3 3 4 4 4 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 3 3 3 4 4 4 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 3 3 3 4 4 4 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 3 3 3 4 4 4 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 3 3 3 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 3 3 3 4 4 4 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 2 2 2 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 4 4 4 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 2 2 2 3 3 3 F F F 29 580-599 2 2 2 3 3 3 F F F Max gen: 3, Max flush: 4 Begin loop 2 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.275487099Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.319597187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.320264801Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.32030647Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.320321273Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.320335816Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204} {"msg":"Crucible 92d4cab0-83c4-4b9f-ba89-c878d111d6a4 has session id: 741c0705-5845-482b-9088-925a4e81a17d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.320933085Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Upstairs opts: Upstairs UUID: 92d4cab0-83c4-4b9f-ba89-c878d111d6a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.320977745Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Crucible stats registered with UUID: 92d4cab0-83c4-4b9f-ba89-c878d111d6a4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.32100243Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:18:50.321024165Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.32156013Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.321586676Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204} {"msg":"92d4cab0-83c4-4b9f-ba89-c878d111d6a4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.321667123Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.321686958Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"0","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.321729217Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"0","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.321756733Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"1","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.321823967Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"1","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.321852993Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"2","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.32188202Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"2","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.321906725Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"io task","client":"2","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.321933241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"io task","client":"0","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.321957646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"io task","client":"1","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"ds_connection connected from Ok(127.0.0.1:49486)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.321982652Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"io task","client":"2","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"ds_connection connected from Ok(127.0.0.1:35325)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.322142866Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"io task","client":"1","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"ds_connection connected from Ok(127.0.0.1:64183)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.322183525Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"io task","client":"0","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.32271838Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"1","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.322749987Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"0","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.323250515Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"2","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.323360969Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"1","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.323381863Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"0","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.323766316Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"2","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324018341Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324050508Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"[0]R flush_numbers[0..12]: [1, 4, 4, 1, 4, 1, 1, 1, 4, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324321676Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"[0]R generation[0..12]: [1, 3, 3, 1, 3, 1, 1, 1, 3, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.32434189Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324356744Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"[1]R flush_numbers[0..12]: [1, 4, 4, 1, 4, 1, 1, 1, 4, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324370457Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"[1]R generation[0..12]: [1, 3, 3, 1, 3, 1, 1, 1, 3, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.32438355Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324397092Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"[2]R flush_numbers[0..12]: [1, 4, 4, 1, 4, 1, 1, 1, 4, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324410225Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"[2]R generation[0..12]: [1, 3, 3, 1, 3, 1, 1, 1, 3, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324423438Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324437001Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324452724Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324465807Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.32447853Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324491153Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"":"downstairs","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:18:50.324503705Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324521719Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"0","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324534692Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"1","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324547395Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"client":"2","":"downstairs","session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324560238Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.32457213Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"92d4cab0-83c4-4b9f-ba89-c878d111d6a4 is now active with session: 741c0705-5845-482b-9088-925a4e81a17d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324584433Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324596025Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204,"session_id":"741c0705-5845-482b-9088-925a4e81a17d"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.324607808Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204} {"msg":"Activated sub_volume 92d4cab0-83c4-4b9f-ba89-c878d111d6a4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:18:50.32461856Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 398 len 8192 data: 2 2 02/30 Write block 6 len 12288 data: 2 2 2 03/30 Read block 3 len 28672 04/30 Write block 138 len 8192 data: 2 2 05/30 Read block 463 len 20480 06/30 Write block 357 len 12288 data: 2 2 2 07/30 Flush 08/30 Flush 09/30 Write block 360 len 24576 data: 2 2 2 2 2 2 10/30 Read block 555 len 32768 11/30 Read block 450 len 12288 12/30 Flush 13/30 Write block 165 len 8192 data: 3 3 14/30 Write block 314 len 8192 data: 2 2 15/30 Read block 564 len 36864 16/30 Write block 157 len 36864 data: 2 2 2 2 2 2 3 3 4 17/30 Read block 371 len 12288 18/30 Write block 90 len 8192 data: 2 2 19/30 Write block 113 len 40960 data: 2 2 2 2 2 2 2 2 2 2 20/30 Read block 235 len 24576 21/30 Read block 348 len 28672 22/30 Read block 146 len 24576 23/30 Read block 29 len 28672 24/30 Read block 340 len 4096 25/30 Read block 93 len 28672 26/30 Read block 96 len 28672 27/30 Read block 516 len 16384 28/30 Write block 567 len 36864 data: 4 3 3 3 3 3 3 3 3 29/30 Read block 244 len 28672 30/30 Read block 19 len 24576 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 2 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 18 18 17 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:18:50.344553291Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1204} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 5 5 5 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 3 3 3 4 4 4 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 3 3 3 4 4 4 T T T 5 100-119 1 1 1 1 1 1 T T T 6 120-139 4 4 4 5 5 5 T T T 7 140-159 1 1 1 1 1 1 T T T 8 160-179 3 3 3 4 4 4 T T T 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 3 3 3 4 4 4 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 3 3 3 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 1 1 1 1 1 1 T T T 16 320-339 3 3 3 4 4 4 F F F 17 340-359 4 4 4 5 5 5 F F F 18 360-379 4 4 4 7 7 7 F F F 19 380-399 4 4 4 5 5 5 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 4 4 4 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 2 2 2 3 3 3 T T T 29 580-599 2 2 2 3 3 3 F F F Max gen: 4, Max flush: 7 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 5 5 5 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 3 3 3 4 4 4 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 5 5 5 8 8 8 F F F 5 100-119 5 5 5 8 8 8 F F F 6 120-139 5 5 5 8 8 8 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 5 5 5 8 8 8 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 3 3 3 4 4 4 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 3 3 3 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 5 5 5 8 8 8 F F F 16 320-339 3 3 3 4 4 4 F F F 17 340-359 4 4 4 5 5 5 F F F 18 360-379 4 4 4 7 7 7 F F F 19 380-399 4 4 4 5 5 5 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 4 4 4 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 5 5 5 8 8 8 F F F 29 580-599 2 2 2 3 3 3 F F F Max gen: 5, Max flush: 8 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.587868554Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.633244245Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.633976923Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.634018952Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.634033535Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.634053939Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212} {"msg":"Crucible 3f78e1f1-c446-445b-a421-c23ae54a6de4 has session id: 2f16c9eb-978a-4f35-a390-81e3df22ee45","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.634544935Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Upstairs opts: Upstairs UUID: 3f78e1f1-c446-445b-a421-c23ae54a6de4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.634610459Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Crucible stats registered with UUID: 3f78e1f1-c446-445b-a421-c23ae54a6de4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.634637485Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:04.63465839Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635217561Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635242726Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212} {"msg":"3f78e1f1-c446-445b-a421-c23ae54a6de4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635292547Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635326614Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"0","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635371544Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"0","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635396469Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"1","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635423055Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"1","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.63544954Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"2","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635476306Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"2","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635501962Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"io task","client":"2","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635528017Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"io task","client":"0","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635556074Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"io task","client":"1","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"ds_connection connected from Ok(127.0.0.1:40944)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.63558855Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"io task","client":"2","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"ds_connection connected from Ok(127.0.0.1:54132)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635743714Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"io task","client":"0","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"ds_connection connected from Ok(127.0.0.1:35179)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.635827882Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"io task","client":"1","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.636399076Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"0","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.636442025Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"1","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.636824897Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"2","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.637177924Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"1","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.63720738Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"0","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.63748681Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"2","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.637813221Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.637835135Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"[0]R flush_numbers[0..12]: [5, 4, 4, 1, 8, 8, 8, 8, 8, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.63813545Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"[0]R generation[0..12]: [4, 3, 3, 1, 5, 5, 5, 5, 5, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638163446Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.6381795Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"[1]R flush_numbers[0..12]: [5, 4, 4, 1, 8, 8, 8, 8, 8, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638194193Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"[1]R generation[0..12]: [4, 3, 3, 1, 5, 5, 5, 5, 5, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638212297Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.63822653Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"[2]R flush_numbers[0..12]: [5, 4, 4, 1, 8, 8, 8, 8, 8, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638239793Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"[2]R generation[0..12]: [4, 3, 3, 1, 5, 5, 5, 5, 5, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638252965Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638266799Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638280651Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638293474Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Next flush: 9","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:19:04.638305947Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638322351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"":"downstairs","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638335323Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638346956Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"0","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638359839Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"1","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638372721Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"client":"2","":"downstairs","session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638385484Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638397196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"3f78e1f1-c446-445b-a421-c23ae54a6de4 is now active with session: 2f16c9eb-978a-4f35-a390-81e3df22ee45","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.63841135Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638423072Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212,"session_id":"2f16c9eb-978a-4f35-a390-81e3df22ee45"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638434875Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212} {"msg":"Activated sub_volume 3f78e1f1-c446-445b-a421-c23ae54a6de4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:04.638445667Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 192 len 16384 data: 2 2 2 2 02/30 Write block 278 len 4096 data: 3 03/30 Read block 223 len 16384 04/30 Read block 321 len 36864 05/30 Write block 53 len 24576 data: 2 2 2 2 2 2 06/30 Write block 194 len 4096 data: 3 07/30 Read block 218 len 4096 08/30 Read block 555 len 16384 09/30 Flush 10/30 Write block 74 len 20480 data: 2 2 2 2 2 11/30 Read block 503 len 28672 12/30 Read block 22 len 20480 13/30 Write block 155 len 8192 data: 2 2 14/30 Read block 548 len 24576 15/30 Write block 489 len 4096 data: 2 16/30 Read block 104 len 36864 17/30 Write block 492 len 12288 data: 2 2 2 18/30 Write block 215 len 40960 data: 2 2 2 2 2 2 2 2 2 2 19/30 Read block 580 len 32768 20/30 Write block 293 len 24576 data: 2 2 2 2 2 2 21/30 Read block 489 len 12288 22/30 Write block 512 len 36864 data: 2 2 2 2 2 2 2 2 2 23/30 Read block 218 len 36864 24/30 Read block 150 len 32768 25/30 Read block 529 len 32768 26/30 Read block 363 len 40960 27/30 Write block 251 len 16384 data: 2 2 2 2 28/30 Write block 526 len 20480 data: 2 2 2 2 2 29/30 Write block 467 len 16384 data: 2 2 2 2 30/30 Write block 512 len 28672 data: 3 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 5 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 20 20 19 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:04.657656752Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1212} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 5 5 5 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 6 6 6 9 9 9 F F F 3 060-079 1 1 1 1 1 1 T T T 4 080-099 5 5 5 8 8 8 F F F 5 100-119 5 5 5 8 8 8 F F F 6 120-139 5 5 5 8 8 8 F F F 7 140-159 5 5 5 8 8 8 T T T 8 160-179 5 5 5 8 8 8 F F F 9 180-199 6 6 6 9 9 9 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 3 3 3 4 4 4 T T T 12 240-259 1 1 1 1 1 1 T T T 13 260-279 6 6 6 9 9 9 F F F 14 280-299 2 2 2 3 3 3 T T T 15 300-319 5 5 5 8 8 8 F F F 16 320-339 3 3 3 4 4 4 F F F 17 340-359 4 4 4 5 5 5 F F F 18 360-379 4 4 4 7 7 7 F F F 19 380-399 4 4 4 5 5 5 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 4 4 4 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 3 3 3 4 4 4 T T T 24 480-499 1 1 1 1 1 1 T T T 25 500-519 1 1 1 1 1 1 T T T 26 520-539 1 1 1 1 1 1 T T T 27 540-559 2 2 2 2 2 2 F F F 28 560-579 5 5 5 8 8 8 F F F 29 580-599 2 2 2 3 3 3 F F F Max gen: 6, Max flush: 9 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 5 5 5 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 6 6 6 9 9 9 F F F 3 060-079 7 7 7 10 10 10 F F F 4 080-099 5 5 5 8 8 8 F F F 5 100-119 5 5 5 8 8 8 F F F 6 120-139 5 5 5 8 8 8 F F F 7 140-159 7 7 7 10 10 10 F F F 8 160-179 5 5 5 8 8 8 F F F 9 180-199 6 6 6 9 9 9 F F F 10 200-219 7 7 7 10 10 10 F F F 11 220-239 7 7 7 10 10 10 F F F 12 240-259 7 7 7 10 10 10 F F F 13 260-279 6 6 6 9 9 9 F F F 14 280-299 7 7 7 10 10 10 F F F 15 300-319 5 5 5 8 8 8 F F F 16 320-339 3 3 3 4 4 4 F F F 17 340-359 4 4 4 5 5 5 F F F 18 360-379 4 4 4 7 7 7 F F F 19 380-399 4 4 4 5 5 5 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 4 4 4 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 7 7 7 10 10 10 F F F 25 500-519 7 7 7 10 10 10 F F F 26 520-539 7 7 7 10 10 10 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 5 5 5 8 8 8 F F F 29 580-599 2 2 2 3 3 3 F F F Max gen: 7, Max flush: 10 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:08.924860267Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:08.967817656Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:08.968259391Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:08.999659197Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.000474053Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.000515471Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.000536606Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.00055235Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220} {"msg":"Crucible bca83203-e4f7-41b0-adc9-e79f1d9825f9 has session id: cab04adc-3f6c-4b17-9692-16e345f4594e","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.001224064Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Upstairs opts: Upstairs UUID: bca83203-e4f7-41b0-adc9-e79f1d9825f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.001256911Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Crucible stats registered with UUID: bca83203-e4f7-41b0-adc9-e79f1d9825f9","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.001271935Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:09.001284577Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.001739646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.001764471Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220} {"msg":"bca83203-e4f7-41b0-adc9-e79f1d9825f9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.0018048Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.001824674Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"0","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.0018534Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"0","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.001867973Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"1","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.001882317Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"1","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.001902911Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"2","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.001929667Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"2","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.001954802Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"io task","client":"2","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.001981138Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"io task","client":"0","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.002030298Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"io task","client":"1","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38076)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.002060565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"io task","client":"2","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47052)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.002087441Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"io task","client":"1","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36969)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.002118597Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"io task","client":"0","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.002767997Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"1","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.002808426Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"2","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.00305566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"0","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.003529722Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"2","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.003570351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"1","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.003602217Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"0","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004227102Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004256289Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"[0]R flush_numbers[0..12]: [5, 4, 9, 10, 8, 8, 8, 10, 8, 9, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004284475Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"[0]R generation[0..12]: [4, 3, 6, 7, 5, 5, 5, 7, 5, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004300098Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004314431Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"[1]R flush_numbers[0..12]: [5, 4, 9, 10, 8, 8, 8, 10, 8, 9, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004328934Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"[1]R generation[0..12]: [4, 3, 6, 7, 5, 5, 5, 7, 5, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004343237Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004357351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"[2]R flush_numbers[0..12]: [5, 4, 9, 10, 8, 8, 8, 10, 8, 9, 10, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004370803Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"[2]R generation[0..12]: [4, 3, 6, 7, 5, 5, 5, 7, 5, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004384246Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004398539Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004415843Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004429916Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004443349Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004471685Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"":"downstairs","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004486348Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004498171Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"0","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:19:09.004511384Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"1","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004531348Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"client":"2","":"downstairs","session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004544291Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004556483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"bca83203-e4f7-41b0-adc9-e79f1d9825f9 is now active with session: cab04adc-3f6c-4b17-9692-16e345f4594e","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004569076Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004580829Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220,"session_id":"cab04adc-3f6c-4b17-9692-16e345f4594e"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004592581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220} {"msg":"Activated sub_volume bca83203-e4f7-41b0-adc9-e79f1d9825f9","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:09.004603513Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 208 len 20480 data: 2 2 2 2 2 02/30 Read block 594 len 4096 03/30 Write block 92 len 8192 data: 2 2 04/30 Read block 444 len 8192 05/30 Write block 390 len 16384 data: 2 2 2 2 06/30 Write block 574 len 24576 data: 4 4 2 2 2 2 07/30 Read block 480 len 20480 08/30 Read block 312 len 16384 09/30 Read block 575 len 4096 10/30 Write block 290 len 4096 data: 2 11/30 Write block 525 len 36864 data: 2 3 3 3 3 3 2 2 2 12/30 Flush 13/30 Write block 484 len 40960 data: 2 2 2 2 2 3 2 2 3 3 14/30 Read block 563 len 16384 15/30 Write block 35 len 20480 data: 2 2 2 3 3 16/30 Write block 556 len 20480 data: 2 2 3 3 3 17/30 Read block 115 len 12288 18/30 Write block 198 len 16384 data: 2 2 2 2 19/30 Read block 105 len 12288 20/30 Write block 44 len 28672 data: 3 2 2 2 2 2 2 21/30 Read block 327 len 8192 22/30 Read block 466 len 16384 23/30 Write block 219 len 12288 data: 3 3 3 24/30 Write block 284 len 28672 data: 2 2 2 2 2 2 3 25/30 Flush 26/30 Write block 489 len 32768 data: 4 3 3 4 4 3 2 2 27/30 Write block 91 len 20480 data: 3 3 3 2 2 28/30 Read block 268 len 40960 29/30 Write block 540 len 36864 data: 2 2 2 2 2 2 2 2 2 30/30 Write block 264 len 40960 data: 2 2 2 2 2 2 2 2 3 3 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 9 Done Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:09.024883179Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1220} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 5 5 5 F F F 1 020-039 8 8 8 12 12 12 F F F 2 040-059 8 8 8 12 12 12 F F F 3 060-079 7 7 7 10 10 10 F F F 4 080-099 8 8 8 11 11 11 T T T 5 100-119 5 5 5 8 8 8 F F F 6 120-139 5 5 5 8 8 8 F F F 7 140-159 7 7 7 10 10 10 F F F 8 160-179 5 5 5 8 8 8 F F F 9 180-199 8 8 8 12 12 12 F F F 10 200-219 8 8 8 12 12 12 F F F 11 220-239 8 8 8 12 12 12 F F F 12 240-259 7 7 7 10 10 10 F F F 13 260-279 6 6 6 9 9 9 T T T 14 280-299 8 8 8 12 12 12 F F F 15 300-319 5 5 5 8 8 8 F F F 16 320-339 3 3 3 4 4 4 F F F 17 340-359 4 4 4 5 5 5 F F F 18 360-379 4 4 4 7 7 7 F F F 19 380-399 8 8 8 11 11 11 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 4 4 4 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 8 8 8 12 12 12 T T T 25 500-519 7 7 7 10 10 10 F F F 26 520-539 8 8 8 11 11 11 F F F 27 540-559 8 8 8 12 12 12 T T T 28 560-579 8 8 8 12 12 12 F F F 29 580-599 2 2 2 3 3 3 F F F Max gen: 8, Max flush: 12 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 5 5 5 F F F 1 020-039 8 8 8 12 12 12 F F F 2 040-059 8 8 8 12 12 12 F F F 3 060-079 7 7 7 10 10 10 F F F 4 080-099 9 9 9 13 13 13 F F F 5 100-119 5 5 5 8 8 8 F F F 6 120-139 5 5 5 8 8 8 F F F 7 140-159 7 7 7 10 10 10 F F F 8 160-179 5 5 5 8 8 8 F F F 9 180-199 8 8 8 12 12 12 F F F 10 200-219 8 8 8 12 12 12 F F F 11 220-239 8 8 8 12 12 12 F F F 12 240-259 7 7 7 10 10 10 F F F 13 260-279 9 9 9 13 13 13 F F F 14 280-299 8 8 8 12 12 12 F F F 15 300-319 5 5 5 8 8 8 F F F 16 320-339 3 3 3 4 4 4 F F F 17 340-359 4 4 4 5 5 5 F F F 18 360-379 4 4 4 7 7 7 F F F 19 380-399 8 8 8 11 11 11 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 3 3 3 4 4 4 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 9 9 9 13 13 13 F F F 25 500-519 7 7 7 10 10 10 F F F 26 520-539 8 8 8 11 11 11 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 8 8 8 12 12 12 F F F 29 580-599 2 2 2 3 3 3 F F F Max gen: 9, Max flush: 13 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.286729779Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.331516243Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.332187608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.332241239Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.332260353Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.332278597Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228} {"msg":"Crucible d51b67bf-d56b-43dc-af3e-8f30a7104b3a has session id: b88be0d4-14c3-4ffd-9357-cb47d8dc3c33","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.332734196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Upstairs opts: Upstairs UUID: d51b67bf-d56b-43dc-af3e-8f30a7104b3a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.332779516Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Crucible stats registered with UUID: d51b67bf-d56b-43dc-af3e-8f30a7104b3a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.332808122Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:13.332829526Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333283424Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333315681Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228} {"msg":"d51b67bf-d56b-43dc-af3e-8f30a7104b3a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333373684Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333399499Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"0","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333429136Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"0","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333444439Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"1","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333458852Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"1","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333472295Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"2","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333492809Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"2","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333515364Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"io task","client":"2","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.33354046Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"io task","client":"0","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333568476Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"io task","client":"1","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"ds_connection connected from Ok(127.0.0.1:33845)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333594121Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"io task","client":"0","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"ds_connection connected from Ok(127.0.0.1:59858)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333623567Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"io task","client":"2","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"ds_connection connected from Ok(127.0.0.1:47350)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.333648873Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"io task","client":"1","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.33432905Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"2","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.334474241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"0","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.334944303Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"2","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.335110128Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"0","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.335632851Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.335710828Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.335898709Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"1","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.336408088Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"1","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"[0]R flush_numbers[0..12]: [5, 12, 12, 10, 13, 8, 8, 10, 8, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.336934162Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"[0]R generation[0..12]: [4, 8, 8, 7, 9, 5, 5, 7, 5, 8, 8, 8]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.33697027Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337012689Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"[1]R flush_numbers[0..12]: [5, 12, 12, 10, 13, 8, 8, 10, 8, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337041325Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"[1]R generation[0..12]: [4, 8, 8, 7, 9, 5, 5, 7, 5, 8, 8, 8]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337069581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337097937Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"[2]R flush_numbers[0..12]: [5, 12, 12, 10, 13, 8, 8, 10, 8, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337126423Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"[2]R generation[0..12]: [4, 8, 8, 7, 9, 5, 5, 7, 5, 8, 8, 8]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337162181Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2025-03-17T22:19:13.337191797Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337225535Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.33725152Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337277196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337302081Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"":"downstairs","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337328497Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337352562Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"0","":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337383809Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"1","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337410195Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"client":"2","":"downstairs","session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337427308Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337439351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"d51b67bf-d56b-43dc-af3e-8f30a7104b3a is now active with session: b88be0d4-14c3-4ffd-9357-cb47d8dc3c33","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337454584Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337466487Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228,"session_id":"b88be0d4-14c3-4ffd-9357-cb47d8dc3c33"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337478279Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228} {"msg":"Activated sub_volume d51b67bf-d56b-43dc-af3e-8f30a7104b3a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:13.337489222Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 531 len 4096 data: 3 02/30 Read block 374 len 24576 03/30 Write block 264 len 20480 data: 3 3 3 3 3 04/30 Write block 523 len 12288 data: 2 2 3 05/30 Write block 122 len 8192 data: 3 2 06/30 Flush 07/30 Write block 263 len 12288 data: 2 4 4 08/30 Write block 433 len 40960 data: 2 2 2 2 2 2 2 2 3 3 09/30 Write block 147 len 24576 data: 2 2 2 2 2 2 10/30 Write block 393 len 20480 data: 3 2 2 2 2 11/30 Write block 339 len 8192 data: 2 2 12/30 Write block 122 len 32768 data: 4 3 2 2 2 2 2 2 13/30 Write block 52 len 8192 data: 2 3 14/30 Write block 63 len 4096 data: 2 15/30 Write block 116 len 40960 data: 3 3 3 3 3 3 5 4 3 3 16/30 Read block 188 len 32768 17/30 Write block 335 len 32768 data: 3 2 2 2 3 3 2 2 18/30 Read block 375 len 16384 19/30 Write block 22 len 12288 data: 2 2 2 20/30 Read block 300 len 8192 21/30 Read block 297 len 36864 22/30 Read block 300 len 12288 23/30 Read block 149 len 36864 24/30 Read block 436 len 16384 25/30 Flush 26/30 Write block 355 len 36864 data: 2 2 3 3 3 3 3 3 3 27/30 Read block 47 len 20480 28/30 Write block 575 len 32768 data: 5 3 3 3 3 2 2 2 29/30 Read block 192 len 20480 30/30 Write block 378 len 16384 data: 2 2 2 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:13.355892472Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1228} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 5 5 5 F F F 1 020-039 10 10 10 15 15 15 F F F 2 040-059 10 10 10 15 15 15 F F F 3 060-079 10 10 10 15 15 15 F F F 4 080-099 9 9 9 13 13 13 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 10 10 10 15 15 15 F F F 7 140-159 10 10 10 15 15 15 F F F 8 160-179 5 5 5 8 8 8 F F F 9 180-199 8 8 8 12 12 12 F F F 10 200-219 8 8 8 12 12 12 F F F 11 220-239 8 8 8 12 12 12 F F F 12 240-259 7 7 7 10 10 10 F F F 13 260-279 10 10 10 15 15 15 F F F 14 280-299 8 8 8 12 12 12 F F F 15 300-319 5 5 5 8 8 8 F F F 16 320-339 10 10 10 15 15 15 F F F 17 340-359 10 10 10 15 15 15 T T T 18 360-379 4 4 4 7 7 7 T T T 19 380-399 10 10 10 15 15 15 T T T 20 400-419 1 1 1 1 1 1 F F F 21 420-439 10 10 10 15 15 15 F F F 22 440-459 10 10 10 15 15 15 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 9 9 9 13 13 13 F F F 25 500-519 7 7 7 10 10 10 F F F 26 520-539 10 10 10 14 14 14 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 8 8 8 12 12 12 T T T 29 580-599 2 2 2 3 3 3 T T T Max gen: 10, Max flush: 15 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 5 5 5 F F F 1 020-039 10 10 10 15 15 15 F F F 2 040-059 10 10 10 15 15 15 F F F 3 060-079 10 10 10 15 15 15 F F F 4 080-099 9 9 9 13 13 13 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 10 10 10 15 15 15 F F F 7 140-159 10 10 10 15 15 15 F F F 8 160-179 5 5 5 8 8 8 F F F 9 180-199 8 8 8 12 12 12 F F F 10 200-219 8 8 8 12 12 12 F F F 11 220-239 8 8 8 12 12 12 F F F 12 240-259 7 7 7 10 10 10 F F F 13 260-279 10 10 10 15 15 15 F F F 14 280-299 8 8 8 12 12 12 F F F 15 300-319 5 5 5 8 8 8 F F F 16 320-339 10 10 10 15 15 15 F F F 17 340-359 11 11 11 16 16 16 F F F 18 360-379 11 11 11 16 16 16 F F F 19 380-399 11 11 11 16 16 16 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 10 10 10 15 15 15 F F F 22 440-459 10 10 10 15 15 15 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 9 9 9 13 13 13 F F F 25 500-519 7 7 7 10 10 10 F F F 26 520-539 10 10 10 14 14 14 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 11 11 11 16 16 16 F F F 29 580-599 11 11 11 16 16 16 F F F Max gen: 11, Max flush: 16 Begin loop 6 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.590493304Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.635047618Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.635766053Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.635831407Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.635857133Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.635876097Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236} {"msg":"Crucible 45d1682c-600b-4a75-93ad-7414639f1da1 has session id: a1f88111-e1c2-4fdd-965a-7e5f094c1520","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.636453331Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Upstairs opts: Upstairs UUID: 45d1682c-600b-4a75-93ad-7414639f1da1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.636544861Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Crucible stats registered with UUID: 45d1682c-600b-4a75-93ad-7414639f1da1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.636572717Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:17.636593711Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.637123206Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.637148961Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236} {"msg":"45d1682c-600b-4a75-93ad-7414639f1da1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.63718946Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.637205143Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"0","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.63723445Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"0","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.637248333Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"1","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.637271878Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"1","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.637296663Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"2","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.63732732Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"2","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.637367258Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.637391413Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.637413578Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"ds_connection connected from Ok(127.0.0.1:49513)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.637437133Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"io task","client":"0","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"ds_connection connected from Ok(127.0.0.1:61463)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.637462409Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"io task","client":"1","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"ds_connection connected from Ok(127.0.0.1:58016)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.637686447Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"io task","client":"2","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.638184985Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"0","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.638339318Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"1","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.638377296Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"2","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.638873173Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"0","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.638902259Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"2","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639150203Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"1","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639431093Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639518783Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"[0]R flush_numbers[0..12]: [5, 15, 15, 15, 13, 15, 15, 15, 8, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.63969229Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"[0]R generation[0..12]: [4, 10, 10, 10, 9, 10, 10, 10, 5, 8, 8, 8]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639713245Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639727948Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"[1]R flush_numbers[0..12]: [5, 15, 15, 15, 13, 15, 15, 15, 8, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639741351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"[1]R generation[0..12]: [4, 10, 10, 10, 9, 10, 10, 10, 5, 8, 8, 8]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639754523Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639768146Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"[2]R flush_numbers[0..12]: [5, 15, 15, 15, 13, 15, 15, 15, 8, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639781449Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"[2]R generation[0..12]: [4, 10, 10, 10, 9, 10, 10, 10, 5, 8, 8, 8]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639794382Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639807905Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639820548Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.63983332Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639845793Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639858336Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.63987748Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639891563Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"0","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639904666Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"1","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639917358Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"client":"2","":"downstairs","session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639930141Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639941994Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"45d1682c-600b-4a75-93ad-7414639f1da1 is now active with session: a1f88111-e1c2-4fdd-965a-7e5f094c1520","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639954337Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639965979Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236,"session_id":"a1f88111-e1c2-4fdd-965a-7e5f094c1520"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639977731Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236} {"msg":"Activated sub_volume 45d1682c-600b-4a75-93ad-7414639f1da1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:17.639988784Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 311 len 4096 data: 2 02/30 Read block 133 len 24576 03/30 Read block 333 len 32768 04/30 Read block 0 len 32768 05/30 Flush 06/30 Flush 07/30 Write block 244 len 16384 data: 2 2 2 2 08/30 Write block 90 len 4096 data: 3 09/30 Read block 299 len 40960 10/30 Read block 98 len 20480 11/30 Read block 169 len 12288 12/30 Read block 542 len 20480 13/30 Read block 193 len 8192 14/30 Read block 210 len 28672 15/30 Write block 527 len 4096 data: 4 16/30 Read block 572 len 8192 17/30 Read block 379 len 8192 18/30 Write block 372 len 20480 data: 2 2 2 2 2 19/30 Write block 517 len 24576 data: 4 4 3 3 2 2 20/30 Read block 213 len 32768 21/30 Write block 6 len 36864 data: 3 3 3 2 2 2 2 2 2 22/30 Write block 51 len 28672 data: 2 3 4 3 3 3 3 23/30 Read block 499 len 4096 24/30 Write block 48 len 20480 data: 3 3 3 3 4 25/30 Read block 159 len 8192 26/30 Write block 33 len 4096 data: 2 27/30 Write block 312 len 8192 data: 2 2 28/30 Read block 354 len 4096 29/30 Write block 252 len 4096 data: 3 30/30 Read block 225 len 24576 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 4 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 23 24 22 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:17.658264347Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1236} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 5 5 5 T T T 1 020-039 10 10 10 15 15 15 T T T 2 040-059 10 10 10 15 15 15 T T T 3 060-079 10 10 10 15 15 15 F F F 4 080-099 9 9 9 13 13 13 T T T 5 100-119 10 10 10 15 15 15 F F F 6 120-139 10 10 10 15 15 15 F F F 7 140-159 10 10 10 15 15 15 F F F 8 160-179 5 5 5 8 8 8 F F F 9 180-199 8 8 8 12 12 12 F F F 10 200-219 8 8 8 12 12 12 F F F 11 220-239 8 8 8 12 12 12 F F F 12 240-259 7 7 7 10 10 10 T T T 13 260-279 10 10 10 15 15 15 F F F 14 280-299 8 8 8 12 12 12 F F F 15 300-319 12 12 12 17 17 17 T T T 16 320-339 10 10 10 15 15 15 F F F 17 340-359 11 11 11 16 16 16 F F F 18 360-379 11 11 11 16 16 16 T T T 19 380-399 11 11 11 16 16 16 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 10 10 10 15 15 15 F F F 22 440-459 10 10 10 15 15 15 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 9 9 9 13 13 13 F F F 25 500-519 7 7 7 10 10 10 T T T 26 520-539 10 10 10 14 14 14 T T T 27 540-559 9 9 9 13 13 13 F F F 28 560-579 11 11 11 16 16 16 F F F 29 580-599 11 11 11 16 16 16 F F F Max gen: 12, Max flush: 17 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 13 13 13 18 18 18 F F F 1 020-039 13 13 13 18 18 18 F F F 2 040-059 13 13 13 18 18 18 F F F 3 060-079 10 10 10 15 15 15 F F F 4 080-099 13 13 13 18 18 18 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 10 10 10 15 15 15 F F F 7 140-159 10 10 10 15 15 15 F F F 8 160-179 5 5 5 8 8 8 F F F 9 180-199 8 8 8 12 12 12 F F F 10 200-219 8 8 8 12 12 12 F F F 11 220-239 8 8 8 12 12 12 F F F 12 240-259 13 13 13 18 18 18 F F F 13 260-279 10 10 10 15 15 15 F F F 14 280-299 8 8 8 12 12 12 F F F 15 300-319 13 13 13 18 18 18 F F F 16 320-339 10 10 10 15 15 15 F F F 17 340-359 11 11 11 16 16 16 F F F 18 360-379 13 13 13 18 18 18 F F F 19 380-399 11 11 11 16 16 16 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 10 10 10 15 15 15 F F F 22 440-459 10 10 10 15 15 15 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 9 9 9 13 13 13 F F F 25 500-519 13 13 13 18 18 18 F F F 26 520-539 13 13 13 18 18 18 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 11 11 11 16 16 16 F F F 29 580-599 11 11 11 16 16 16 F F F Max gen: 13, Max flush: 18 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.913170039Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.957745547Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.95850018Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.958555792Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.958571805Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.95859398Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244} {"msg":"Crucible b300f50c-b7db-458b-af02-cb8d0039fca9 has session id: 8c1dd8d5-d2a1-42a2-b060-50f3bb329f59","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.95924282Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Upstairs opts: Upstairs UUID: b300f50c-b7db-458b-af02-cb8d0039fca9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.95928983Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Crucible stats registered with UUID: b300f50c-b7db-458b-af02-cb8d0039fca9","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.959314715Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:21.95933592Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.959721043Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.9597541Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244} {"msg":"b300f50c-b7db-458b-af02-cb8d0039fca9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.959808562Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.959834358Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"0","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.959863074Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"0","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.959878077Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"1","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.95989216Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"1","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.959905313Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"2","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.959923697Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"2","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.959946682Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.959972588Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.959996233Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"ds_connection connected from Ok(127.0.0.1:43128)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.960039112Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"io task","client":"0","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:21.960067938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:21.960097955Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"ds_connection connected from Ok(127.0.0.1:59658)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.960131612Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"io task","client":"1","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:21.960163549Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.96021566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.960730011Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"0","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.960759157Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"1","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.961443805Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"0","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.961472881Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"1","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.962061178Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:21.962086704Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.961683918Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"ds_connection connected from Ok(127.0.0.1:59326)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.961818987Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"io task","client":"2","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.962990289Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"2","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.963565714Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"2","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"[0]R flush_numbers[0..12]: [18, 18, 18, 15, 18, 15, 15, 15, 8, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964127725Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"[0]R generation[0..12]: [13, 13, 13, 10, 13, 10, 10, 10, 5, 8, 8, 8]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964169614Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964190058Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"[1]R flush_numbers[0..12]: [18, 18, 18, 15, 18, 15, 15, 15, 8, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964207592Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"[1]R generation[0..12]: [13, 13, 13, 10, 13, 10, 10, 10, 5, 8, 8, 8]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964224586Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.96424209Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"[2]R flush_numbers[0..12]: [18, 18, 18, 15, 18, 15, 15, 15, 8, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964259113Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"[2]R generation[0..12]: [13, 13, 13, 10, 13, 10, 10, 10, 5, 8, 8, 8]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964275597Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.96429269Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964308404Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964324537Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Next flush: 19","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964340101Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964355514Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"":"downstairs","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:19:31.964371607Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964391902Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"0","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964408155Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"1","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.96442779Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"client":"2","":"downstairs","session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964444243Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964459066Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"b300f50c-b7db-458b-af02-cb8d0039fca9 is now active with session: 8c1dd8d5-d2a1-42a2-b060-50f3bb329f59","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.96447464Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964489173Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244,"session_id":"8c1dd8d5-d2a1-42a2-b060-50f3bb329f59"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964503856Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244} {"msg":"Activated sub_volume b300f50c-b7db-458b-af02-cb8d0039fca9","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:31.964518699Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 172 len 24576 data: 2 2 2 2 2 2 02/30 Write block 39 len 40960 data: 4 3 3 3 3 4 3 3 3 4 03/30 Write block 217 len 28672 data: 3 3 4 4 4 3 3 04/30 Flush 05/30 Write block 184 len 4096 data: 2 06/30 Read block 181 len 36864 07/30 Flush 08/30 Write block 19 len 24576 data: 2 2 2 3 3 3 09/30 Write block 407 len 24576 data: 2 2 2 2 2 2 10/30 Read block 56 len 8192 11/30 Read block 555 len 36864 12/30 Write block 154 len 12288 data: 2 3 3 13/30 Write block 489 len 24576 data: 5 4 4 5 5 4 14/30 Read block 431 len 16384 15/30 Write block 83 len 24576 data: 2 2 2 2 2 2 16/30 Write block 522 len 8192 data: 3 3 17/30 Write block 135 len 4096 data: 2 18/30 Flush 19/30 Write block 522 len 32768 data: 4 4 3 4 4 5 4 4 20/30 Read block 283 len 12288 21/30 Write block 584 len 40960 data: 3 3 2 2 2 2 2 2 2 2 22/30 Read block 262 len 40960 23/30 Write block 176 len 20480 data: 3 3 2 2 2 24/30 Write block 62 len 4096 data: 2 25/30 Read block 446 len 36864 26/30 Read block 213 len 28672 27/30 Write block 52 len 36864 data: 5 5 4 4 4 4 3 2 2 28/30 Read block 63 len 36864 29/30 Write block 439 len 20480 data: 3 3 4 4 3 30/30 Read block 352 len 40960 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 8 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 12 11 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:31.985780697Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1244} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 21 21 21 F F F 1 020-039 14 14 14 21 21 21 F F F 2 040-059 14 14 14 19 19 19 T T T 3 060-079 10 10 10 15 15 15 T T T 4 080-099 14 14 14 21 21 21 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 14 14 14 21 21 21 F F F 7 140-159 14 14 14 21 21 21 F F F 8 160-179 14 14 14 19 19 19 T T T 9 180-199 14 14 14 20 20 20 T T T 10 200-219 14 14 14 19 19 19 F F F 11 220-239 14 14 14 19 19 19 F F F 12 240-259 13 13 13 18 18 18 F F F 13 260-279 10 10 10 15 15 15 F F F 14 280-299 8 8 8 12 12 12 F F F 15 300-319 13 13 13 18 18 18 F F F 16 320-339 10 10 10 15 15 15 F F F 17 340-359 11 11 11 16 16 16 F F F 18 360-379 13 13 13 18 18 18 F F F 19 380-399 11 11 11 16 16 16 F F F 20 400-419 14 14 14 21 21 21 F F F 21 420-439 10 10 10 15 15 15 T T T 22 440-459 10 10 10 15 15 15 T T T 23 460-479 7 7 7 10 10 10 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 13 13 13 18 18 18 F F F 26 520-539 14 14 14 21 21 21 T T T 27 540-559 9 9 9 13 13 13 F F F 28 560-579 11 11 11 16 16 16 F F F 29 580-599 11 11 11 16 16 16 T T T Max gen: 14, Max flush: 21 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 21 21 21 F F F 1 020-039 14 14 14 21 21 21 F F F 2 040-059 15 15 15 22 22 22 F F F 3 060-079 15 15 15 22 22 22 F F F 4 080-099 14 14 14 21 21 21 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 14 14 14 21 21 21 F F F 7 140-159 14 14 14 21 21 21 F F F 8 160-179 15 15 15 22 22 22 F F F 9 180-199 15 15 15 22 22 22 F F F 10 200-219 14 14 14 19 19 19 F F F 11 220-239 14 14 14 19 19 19 F F F 12 240-259 13 13 13 18 18 18 F F F 13 260-279 10 10 10 15 15 15 F F F 14 280-299 8 8 8 12 12 12 F F F 15 300-319 13 13 13 18 18 18 F F F 16 320-339 10 10 10 15 15 15 F F F 17 340-359 11 11 11 16 16 16 F F F 18 360-379 13 13 13 18 18 18 F F F 19 380-399 11 11 11 16 16 16 F F F 20 400-419 14 14 14 21 21 21 F F F 21 420-439 15 15 15 22 22 22 F F F 22 440-459 15 15 15 22 22 22 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 14 14 14 21 21 21 F F F 25 500-519 13 13 13 18 18 18 F F F 26 520-539 15 15 15 22 22 22 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 11 11 11 16 16 16 F F F 29 580-599 15 15 15 22 22 22 F F F Max gen: 15, Max flush: 22 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.244901268Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.288349703Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.289027199Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.289068928Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.289084381Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.289098565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252} {"msg":"Crucible 9855c36c-9e9d-4c94-a65c-7113e64e5205 has session id: 03027e77-11cc-4f96-a553-0f3b47b05434","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.289621077Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Upstairs opts: Upstairs UUID: 9855c36c-9e9d-4c94-a65c-7113e64e5205, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.289664967Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Crucible stats registered with UUID: 9855c36c-9e9d-4c94-a65c-7113e64e5205","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.289688222Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:36.289708906Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290103291Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290127376Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252} {"msg":"9855c36c-9e9d-4c94-a65c-7113e64e5205 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290186809Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290213755Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"0","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290247162Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"0","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290262196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"1","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290276679Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"1","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290290142Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"2","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290308926Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"2","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290332511Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290356746Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290382462Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"ds_connection connected from Ok(127.0.0.1:42496)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290408457Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"io task","client":"0","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:36.290437193Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"ds_connection connected from Ok(127.0.0.1:63766)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290462969Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"io task","client":"1","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:36.290490575Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:36.290528383Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.290641117Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.291182284Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"0","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.291222133Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"1","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.291800187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"0","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.291908641Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"1","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.292355257Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:36.292440635Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.291518236Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"ds_connection connected from Ok(127.0.0.1:61053)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.291688353Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"io task","client":"2","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.292922109Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"2","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.293538832Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"2","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"[0]R flush_numbers[0..12]: [21, 21, 22, 22, 21, 15, 21, 21, 22, 22, 19, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294168098Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"[0]R generation[0..12]: [14, 14, 15, 15, 14, 10, 14, 14, 15, 15, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294189092Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294204396Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"[1]R flush_numbers[0..12]: [21, 21, 22, 22, 21, 15, 21, 21, 22, 22, 19, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294218859Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"[1]R generation[0..12]: [14, 14, 15, 15, 14, 10, 14, 14, 15, 15, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294237783Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294252176Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"[2]R flush_numbers[0..12]: [21, 21, 22, 22, 21, 15, 21, 21, 22, 22, 19, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294265389Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"[2]R generation[0..12]: [14, 14, 15, 15, 14, 10, 14, 14, 15, 15, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294278452Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294291905Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294304457Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.29431712Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Next flush: 23","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294329733Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294342266Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"":"downstairs","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294355138Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294366651Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"0","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294379584Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"1","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294392476Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"client":"2","":"downstairs","session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294413781Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294425804Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"9855c36c-9e9d-4c94-a65c-7113e64e5205 is now active with session: 03027e77-11cc-4f96-a553-0f3b47b05434","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294452509Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294465702Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252,"session_id":"03027e77-11cc-4f96-a553-0f3b47b05434"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294477965Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252} {"msg":"Activated sub_volume 9855c36c-9e9d-4c94-a65c-7113e64e5205","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:46.294488767Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 325 len 32768 data: 2 2 2 2 2 2 3 3 02/30 Write block 362 len 36864 data: 4 4 3 3 2 2 2 2 2 03/30 Read block 66 len 8192 04/30 Write block 515 len 12288 data: 4 4 5 05/30 Write block 173 len 28672 data: 3 3 3 4 4 3 3 06/30 Read block 205 len 4096 07/30 Read block 315 len 24576 08/30 Write block 408 len 40960 data: 3 3 3 3 3 2 2 2 2 2 09/30 Write block 436 len 8192 data: 3 3 10/30 Read block 44 len 36864 11/30 Flush 12/30 Write block 12 len 4096 data: 3 13/30 Read block 464 len 40960 14/30 Read block 536 len 36864 15/30 Read block 347 len 16384 16/30 Read block 81 len 32768 17/30 Read block 18 len 12288 18/30 Write block 250 len 4096 data: 2 19/30 Read block 261 len 4096 20/30 Write block 188 len 40960 data: 2 2 2 2 3 3 4 3 2 2 21/30 Write block 257 len 28672 data: 2 2 2 2 2 2 3 22/30 Write block 353 len 32768 data: 2 2 3 3 4 4 4 4 23/30 Write block 46 len 20480 data: 4 4 5 4 4 24/30 Read block 122 len 24576 25/30 Flush 26/30 Flush 27/30 Write block 72 len 28672 data: 2 2 3 3 3 3 3 28/30 Write block 472 len 36864 data: 2 2 2 2 2 2 2 2 2 29/30 Read block 560 len 28672 30/30 Read block 118 len 28672 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:46.314631153Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1252} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 24 24 24 F F F 1 020-039 14 14 14 21 21 21 F F F 2 040-059 16 16 16 24 24 24 F F F 3 060-079 15 15 15 22 22 22 T T T 4 080-099 14 14 14 21 21 21 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 14 14 14 21 21 21 F F F 7 140-159 14 14 14 21 21 21 F F F 8 160-179 16 16 16 23 23 23 F F F 9 180-199 16 16 16 24 24 24 F F F 10 200-219 14 14 14 19 19 19 F F F 11 220-239 14 14 14 19 19 19 F F F 12 240-259 16 16 16 24 24 24 F F F 13 260-279 16 16 16 24 24 24 F F F 14 280-299 8 8 8 12 12 12 F F F 15 300-319 13 13 13 18 18 18 F F F 16 320-339 16 16 16 23 23 23 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 16 16 16 24 24 24 F F F 19 380-399 11 11 11 16 16 16 F F F 20 400-419 16 16 16 23 23 23 F F F 21 420-439 16 16 16 23 23 23 F F F 22 440-459 15 15 15 22 22 22 F F F 23 460-479 7 7 7 10 10 10 T T T 24 480-499 14 14 14 21 21 21 T T T 25 500-519 16 16 16 23 23 23 F F F 26 520-539 15 15 15 22 22 22 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 11 11 11 16 16 16 F F F 29 580-599 15 15 15 22 22 22 F F F Max gen: 16, Max flush: 24 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 24 24 24 F F F 1 020-039 14 14 14 21 21 21 F F F 2 040-059 16 16 16 24 24 24 F F F 3 060-079 17 17 17 25 25 25 F F F 4 080-099 14 14 14 21 21 21 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 14 14 14 21 21 21 F F F 7 140-159 14 14 14 21 21 21 F F F 8 160-179 16 16 16 23 23 23 F F F 9 180-199 16 16 16 24 24 24 F F F 10 200-219 14 14 14 19 19 19 F F F 11 220-239 14 14 14 19 19 19 F F F 12 240-259 16 16 16 24 24 24 F F F 13 260-279 16 16 16 24 24 24 F F F 14 280-299 8 8 8 12 12 12 F F F 15 300-319 13 13 13 18 18 18 F F F 16 320-339 16 16 16 23 23 23 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 16 16 16 24 24 24 F F F 19 380-399 11 11 11 16 16 16 F F F 20 400-419 16 16 16 23 23 23 F F F 21 420-439 16 16 16 23 23 23 F F F 22 440-459 15 15 15 22 22 22 F F F 23 460-479 17 17 17 25 25 25 F F F 24 480-499 17 17 17 25 25 25 F F F 25 500-519 16 16 16 23 23 23 F F F 26 520-539 15 15 15 22 22 22 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 11 11 11 16 16 16 F F F 29 580-599 15 15 15 22 22 22 F F F Max gen: 17, Max flush: 25 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.544365324Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.588936501Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.58976533Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.589816951Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.589842057Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.589862151Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260} {"msg":"Crucible 5a8f18f2-dee3-42a1-adbc-d0a0b9f38eb0 has session id: 54598648-dbca-4b75-8bdd-d5a5c901ffd2","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.590406548Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Upstairs opts: Upstairs UUID: 5a8f18f2-dee3-42a1-adbc-d0a0b9f38eb0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.590550739Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Crucible stats registered with UUID: 5a8f18f2-dee3-42a1-adbc-d0a0b9f38eb0","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.590573354Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:50.590586797Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591070892Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591104439Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260} {"msg":"5a8f18f2-dee3-42a1-adbc-d0a0b9f38eb0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591170533Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591196879Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"0","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591226665Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"0","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591242459Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"1","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591256792Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"1","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591269935Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"2","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591286938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"2","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591361705Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591391851Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591419527Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37023)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.591444512Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"io task","client":"0","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:50.591473979Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:50.591501835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:34961)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.59152713Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"io task","client":"1","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:19:50.591558077Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.59161802Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.592114187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"0","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.592144263Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"1","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.59273267Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"1","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.592844665Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"0","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.593302193Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:19:50.593398734Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.592819901Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:64190)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.592898878Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"io task","client":"2","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.594077402Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"2","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.59462642Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"2","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"[0]R flush_numbers[0..12]: [24, 21, 24, 25, 21, 15, 21, 21, 23, 24, 19, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595223769Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"[0]R generation[0..12]: [16, 14, 16, 17, 14, 10, 14, 14, 16, 16, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595248214Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595264048Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"[1]R flush_numbers[0..12]: [24, 21, 24, 25, 21, 15, 21, 21, 23, 24, 19, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595278021Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"[1]R generation[0..12]: [16, 14, 16, 17, 14, 10, 14, 14, 16, 16, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595291554Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595305437Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"[2]R flush_numbers[0..12]: [24, 21, 24, 25, 21, 15, 21, 21, 23, 24, 19, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.5953186Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"[2]R generation[0..12]: [16, 14, 16, 17, 14, 10, 14, 14, 16, 16, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595332092Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595350187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595363219Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595375962Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595388765Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595401707Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"":"downstairs","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.59541466Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595426283Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"0","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595439286Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"1","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:20:00.595451938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"client":"2","":"downstairs","session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595470372Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595482255Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"5a8f18f2-dee3-42a1-adbc-d0a0b9f38eb0 is now active with session: 54598648-dbca-4b75-8bdd-d5a5c901ffd2","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595494778Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.59550759Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260,"session_id":"54598648-dbca-4b75-8bdd-d5a5c901ffd2"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595519713Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260} {"msg":"Activated sub_volume 5a8f18f2-dee3-42a1-adbc-d0a0b9f38eb0","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:00.595530415Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 36 len 24576 data: 3 3 4 5 4 4 02/30 Read block 288 len 28672 03/30 Read block 354 len 4096 04/30 Read block 404 len 12288 05/30 Read block 335 len 36864 06/30 Read block 55 len 16384 07/30 Write block 165 len 8192 data: 5 4 08/30 Write block 424 len 20480 data: 2 2 2 2 3 09/30 Write block 487 len 36864 data: 3 3 6 5 5 6 6 5 3 10/30 Write block 265 len 16384 data: 5 4 4 4 11/30 Write block 185 len 20480 data: 2 2 2 3 3 12/30 Read block 7 len 12288 13/30 Write block 329 len 36864 data: 3 3 4 4 3 3 4 3 3 14/30 Read block 317 len 28672 15/30 Write block 330 len 24576 data: 4 5 5 4 4 5 16/30 Read block 542 len 40960 17/30 Write block 589 len 12288 data: 3 3 3 18/30 Write block 493 len 32768 data: 7 6 4 3 2 2 2 2 19/30 Write block 427 len 36864 data: 3 4 3 3 2 2 3 3 3 20/30 Write block 523 len 24576 data: 5 4 5 5 6 5 21/30 Read block 224 len 20480 22/30 Read block 81 len 32768 23/30 Read block 324 len 36864 24/30 Flush 25/30 Write block 130 len 40960 data: 2 2 2 2 2 3 2 2 3 3 26/30 Write block 142 len 32768 data: 2 2 2 2 2 3 3 3 27/30 Read block 270 len 36864 28/30 Write block 62 len 36864 data: 3 3 2 2 2 2 2 2 2 29/30 Read block 386 len 40960 30/30 Read block 539 len 36864 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:00.618086242Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1260} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 24 24 24 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 18 18 18 26 26 26 F F F 3 060-079 17 17 17 25 25 25 T T T 4 080-099 14 14 14 21 21 21 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 14 14 14 21 21 21 T T T 7 140-159 14 14 14 21 21 21 T T T 8 160-179 18 18 18 26 26 26 F F F 9 180-199 18 18 18 26 26 26 F F F 10 200-219 14 14 14 19 19 19 F F F 11 220-239 14 14 14 19 19 19 F F F 12 240-259 16 16 16 24 24 24 F F F 13 260-279 18 18 18 26 26 26 F F F 14 280-299 8 8 8 12 12 12 F F F 15 300-319 13 13 13 18 18 18 F F F 16 320-339 18 18 18 26 26 26 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 16 16 16 24 24 24 F F F 19 380-399 11 11 11 16 16 16 F F F 20 400-419 16 16 16 23 23 23 F F F 21 420-439 18 18 18 26 26 26 F F F 22 440-459 15 15 15 22 22 22 F F F 23 460-479 17 17 17 25 25 25 F F F 24 480-499 18 18 18 26 26 26 F F F 25 500-519 18 18 18 26 26 26 F F F 26 520-539 18 18 18 26 26 26 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 11 11 11 16 16 16 F F F 29 580-599 18 18 18 26 26 26 F F F Max gen: 18, Max flush: 26 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 24 24 24 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 18 18 18 26 26 26 F F F 3 060-079 19 19 19 27 27 27 F F F 4 080-099 14 14 14 21 21 21 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 19 19 19 27 27 27 F F F 7 140-159 19 19 19 27 27 27 F F F 8 160-179 18 18 18 26 26 26 F F F 9 180-199 18 18 18 26 26 26 F F F 10 200-219 14 14 14 19 19 19 F F F 11 220-239 14 14 14 19 19 19 F F F 12 240-259 16 16 16 24 24 24 F F F 13 260-279 18 18 18 26 26 26 F F F 14 280-299 8 8 8 12 12 12 F F F 15 300-319 13 13 13 18 18 18 F F F 16 320-339 18 18 18 26 26 26 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 16 16 16 24 24 24 F F F 19 380-399 11 11 11 16 16 16 F F F 20 400-419 16 16 16 23 23 23 F F F 21 420-439 18 18 18 26 26 26 F F F 22 440-459 15 15 15 22 22 22 F F F 23 460-479 17 17 17 25 25 25 F F F 24 480-499 18 18 18 26 26 26 F F F 25 500-519 18 18 18 26 26 26 F F F 26 520-539 18 18 18 26 26 26 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 11 11 11 16 16 16 F F F 29 580-599 18 18 18 26 26 26 F F F Max gen: 19, Max flush: 27 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.854143026Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.898772335Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.899484759Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.899532969Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.899550673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.899568497Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268} {"msg":"Crucible c3f62871-2b6e-4e1e-aace-f3097b6013e8 has session id: aed5b8b0-4406-4508-9f2a-3867f27f0051","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900077587Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Upstairs opts: Upstairs UUID: c3f62871-2b6e-4e1e-aace-f3097b6013e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900127968Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Crucible stats registered with UUID: c3f62871-2b6e-4e1e-aace-f3097b6013e8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900155944Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:04.900178849Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900676066Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900700721Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268} {"msg":"c3f62871-2b6e-4e1e-aace-f3097b6013e8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900762965Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900781279Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"0","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900808505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"0","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900822098Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"1","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900836031Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"1","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900852234Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"2","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.90087815Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"2","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900904055Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"io task","client":"2","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900929611Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"io task","client":"0","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.900950805Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"io task","client":"1","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"ds_connection connected from Ok(127.0.0.1:38303)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.901007247Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"io task","client":"2","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"ds_connection connected from Ok(127.0.0.1:36958)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.901038674Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"io task","client":"0","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"ds_connection connected from Ok(127.0.0.1:57629)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.901067591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"io task","client":"1","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.901809231Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"2","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.901845348Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"0","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.902470393Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"0","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.902512792Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"2","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.903084136Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.903175686Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.903598227Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"1","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904096154Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"1","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"[0]R flush_numbers[0..12]: [24, 26, 26, 27, 21, 15, 27, 27, 26, 26, 19, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904601943Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"[0]R generation[0..12]: [16, 18, 18, 19, 14, 10, 19, 19, 18, 18, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904621798Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904636371Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"[1]R flush_numbers[0..12]: [24, 26, 26, 27, 21, 15, 27, 27, 26, 26, 19, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904649744Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"[1]R generation[0..12]: [16, 18, 18, 19, 14, 10, 19, 19, 18, 18, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904667097Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.90468105Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"[2]R flush_numbers[0..12]: [24, 26, 26, 27, 21, 15, 27, 27, 26, 26, 19, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904694703Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"[2]R generation[0..12]: [16, 18, 18, 19, 14, 10, 19, 19, 18, 18, 14, 14]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904708256Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904721899Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904734432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904747025Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904759387Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.90477187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904791584Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904803357Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"0","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904816099Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"1","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904828632Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"client":"2","":"downstairs","session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904841255Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904853197Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"c3f62871-2b6e-4e1e-aace-f3097b6013e8 is now active with session: aed5b8b0-4406-4508-9f2a-3867f27f0051","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.9048673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904879513Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268,"session_id":"aed5b8b0-4406-4508-9f2a-3867f27f0051"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904891216Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268} {"msg":"Activated sub_volume c3f62871-2b6e-4e1e-aace-f3097b6013e8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:04.904902168Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 70 len 24576 data: 3 2 3 3 4 4 02/30 Write block 242 len 8192 data: 2 2 03/30 Flush 04/30 Write block 159 len 28672 data: 3 3 3 3 4 4 6 05/30 Write block 151 len 4096 data: 3 06/30 Read block 29 len 32768 07/30 Write block 499 len 28672 data: 3 3 2 2 2 2 2 08/30 Write block 281 len 24576 data: 2 2 2 3 3 3 09/30 Read block 314 len 8192 10/30 Read block 122 len 20480 11/30 Write block 73 len 16384 data: 4 5 5 4 12/30 Read block 262 len 20480 13/30 Read block 174 len 40960 14/30 Read block 590 len 32768 15/30 Write block 46 len 28672 data: 5 5 6 5 5 4 6 16/30 Write block 259 len 32768 data: 3 3 3 3 4 5 6 5 17/30 Read block 505 len 32768 18/30 Write block 518 len 4096 data: 5 19/30 Write block 233 len 32768 data: 2 3 3 3 3 3 2 2 20/30 Write block 161 len 20480 data: 4 4 5 5 7 21/30 Write block 501 len 12288 data: 3 3 3 22/30 Write block 154 len 32768 data: 3 4 4 3 3 4 4 5 23/30 Read block 558 len 20480 24/30 Read block 481 len 12288 25/30 Read block 230 len 8192 26/30 Read block 286 len 12288 27/30 Write block 296 len 40960 data: 3 3 3 2 2 2 2 2 2 2 28/30 Write block 449 len 24576 data: 2 2 2 2 2 2 29/30 Write block 56 len 36864 data: 5 5 4 3 3 2 4 4 3 30/30 Read block 190 len 32768 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 7 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 9 Done Sent Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 27 25 25 77 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:04.924270877Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1268} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 24 24 24 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 18 18 18 26 26 26 T T T 3 060-079 20 20 20 28 28 28 T T T 4 080-099 14 14 14 21 21 21 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 19 19 19 27 27 27 F F F 7 140-159 19 19 19 27 27 27 T T T 8 160-179 18 18 18 26 26 26 T T T 9 180-199 18 18 18 26 26 26 F F F 10 200-219 14 14 14 19 19 19 F F F 11 220-239 14 14 14 19 19 19 T T T 12 240-259 20 20 20 28 28 28 T T T 13 260-279 18 18 18 26 26 26 T T T 14 280-299 8 8 8 12 12 12 T T T 15 300-319 13 13 13 18 18 18 T T T 16 320-339 18 18 18 26 26 26 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 16 16 16 24 24 24 F F F 19 380-399 11 11 11 16 16 16 F F F 20 400-419 16 16 16 23 23 23 F F F 21 420-439 18 18 18 26 26 26 F F F 22 440-459 15 15 15 22 22 22 T T T 23 460-479 17 17 17 25 25 25 F F F 24 480-499 18 18 18 26 26 26 T T T 25 500-519 18 18 18 26 26 26 T T T 26 520-539 18 18 18 26 26 26 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 11 11 11 16 16 16 F F F 29 580-599 18 18 18 26 26 26 F F F Max gen: 20, Max flush: 28 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 24 24 24 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 21 21 21 29 29 29 F F F 3 060-079 21 21 21 29 29 29 F F F 4 080-099 14 14 14 21 21 21 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 19 19 19 27 27 27 F F F 7 140-159 21 21 21 29 29 29 F F F 8 160-179 21 21 21 29 29 29 F F F 9 180-199 18 18 18 26 26 26 F F F 10 200-219 14 14 14 19 19 19 F F F 11 220-239 21 21 21 29 29 29 F F F 12 240-259 21 21 21 29 29 29 F F F 13 260-279 21 21 21 29 29 29 F F F 14 280-299 21 21 21 29 29 29 F F F 15 300-319 21 21 21 29 29 29 F F F 16 320-339 18 18 18 26 26 26 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 16 16 16 24 24 24 F F F 19 380-399 11 11 11 16 16 16 F F F 20 400-419 16 16 16 23 23 23 F F F 21 420-439 18 18 18 26 26 26 F F F 22 440-459 21 21 21 29 29 29 F F F 23 460-479 17 17 17 25 25 25 F F F 24 480-499 21 21 21 29 29 29 F F F 25 500-519 21 21 21 29 29 29 F F F 26 520-539 18 18 18 26 26 26 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 11 11 11 16 16 16 F F F 29 580-599 18 18 18 26 26 26 F F F Max gen: 21, Max flush: 29 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.197355591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:19.239463237Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.239837658Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.270960763Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.271611354Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.271648372Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.271664785Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.271681869Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276} {"msg":"Crucible e38b381f-1ab3-443c-bfc2-7abc798a1a32 has session id: 6c9baed4-6e07-4286-b933-f2786c214665","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272225666Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Upstairs opts: Upstairs UUID: e38b381f-1ab3-443c-bfc2-7abc798a1a32, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272268175Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Crucible stats registered with UUID: e38b381f-1ab3-443c-bfc2-7abc798a1a32","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272291981Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:19.272312765Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272689306Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272717952Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276} {"msg":"e38b381f-1ab3-443c-bfc2-7abc798a1a32 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272757341Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272773704Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"0","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.27280069Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"0","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272814643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"1","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272828736Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"1","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272841739Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"2","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272855762Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"2","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272868585Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"io task","client":"2","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272881698Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"io task","client":"0","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.272894831Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"io task","client":"1","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"ds_connection connected from Ok(127.0.0.1:45639)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.27298244Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"io task","client":"2","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"ds_connection connected from Ok(127.0.0.1:58055)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.273014707Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"io task","client":"1","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"ds_connection connected from Ok(127.0.0.1:46849)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.273184823Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"io task","client":"0","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.273834853Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"1","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.273882044Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"2","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.2739125Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"0","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.274399886Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"2","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.274428372Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"1","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.274685367Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"0","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.274988082Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275080992Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"[0]R flush_numbers[0..12]: [24, 26, 29, 29, 21, 15, 27, 29, 29, 26, 19, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275199178Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"[0]R generation[0..12]: [16, 18, 21, 21, 14, 10, 19, 21, 21, 18, 14, 21]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275218962Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275234506Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"[1]R flush_numbers[0..12]: [24, 26, 29, 29, 21, 15, 27, 29, 29, 26, 19, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275248478Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"[1]R generation[0..12]: [16, 18, 21, 21, 14, 10, 19, 21, 21, 18, 14, 21]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275261571Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275274874Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"[2]R flush_numbers[0..12]: [24, 26, 29, 29, 21, 15, 27, 29, 29, 26, 19, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275288067Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"[2]R generation[0..12]: [16, 18, 21, 21, 14, 10, 19, 21, 21, 18, 14, 21]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275304801Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275318834Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275331596Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275344169Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275356892Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275375556Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"":"downstairs","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275388509Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275400151Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"0","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275414144Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"1","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275426907Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"client":"2","":"downstairs","session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.27543955Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275451893Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"e38b381f-1ab3-443c-bfc2-7abc798a1a32 is now active with session: 6c9baed4-6e07-4286-b933-f2786c214665","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275464565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275476118Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276,"session_id":"6c9baed4-6e07-4286-b933-f2786c214665"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.27548783Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276} {"msg":"Activated sub_volume e38b381f-1ab3-443c-bfc2-7abc798a1a32","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:19.275498583Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 179 len 40960 data: 4 3 2 2 2 3 3 3 3 4 02/30 Read block 482 len 20480 03/30 Read block 441 len 20480 04/30 Flush 05/30 Write block 463 len 12288 data: 3 3 3 06/30 Read block 320 len 4096 07/30 Read block 411 len 12288 08/30 Flush 09/30 Flush 10/30 Flush 11/30 Write block 263 len 32768 data: 5 6 7 6 5 5 3 3 12/30 Write block 503 len 32768 data: 4 3 3 2 2 2 2 2 13/30 Write block 4 len 36864 data: 2 2 4 4 4 3 3 3 4 14/30 Read block 14 len 32768 15/30 Read block 116 len 40960 16/30 Write block 165 len 24576 data: 8 5 2 2 2 2 17/30 Read block 471 len 28672 18/30 Read block 414 len 36864 19/30 Write block 560 len 16384 data: 4 4 4 4 20/30 Write block 249 len 12288 data: 2 3 3 21/30 Write block 206 len 8192 data: 2 2 22/30 Flush 23/30 Write block 453 len 40960 data: 3 3 2 2 2 2 2 2 3 3 24/30 Read block 500 len 8192 25/30 Write block 180 len 4096 data: 4 26/30 Read block 327 len 20480 27/30 Write block 269 len 4096 data: 4 28/30 Write block 395 len 24576 data: 3 3 3 3 3 2 29/30 Read block 527 len 24576 30/30 Read block 425 len 4096 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:19.295342194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1276} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 34 34 34 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 21 21 21 29 29 29 F F F 3 060-079 21 21 21 29 29 29 F F F 4 080-099 14 14 14 21 21 21 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 19 19 19 27 27 27 F F F 7 140-159 21 21 21 29 29 29 F F F 8 160-179 22 22 22 34 34 34 F F F 9 180-199 22 22 22 30 30 30 T T T 10 200-219 22 22 22 34 34 34 F F F 11 220-239 21 21 21 29 29 29 F F F 12 240-259 22 22 22 34 34 34 F F F 13 260-279 22 22 22 34 34 34 T T T 14 280-299 21 21 21 29 29 29 F F F 15 300-319 21 21 21 29 29 29 F F F 16 320-339 18 18 18 26 26 26 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 16 16 16 24 24 24 F F F 19 380-399 11 11 11 16 16 16 T T T 20 400-419 16 16 16 23 23 23 T T T 21 420-439 18 18 18 26 26 26 F F F 22 440-459 21 21 21 29 29 29 T T T 23 460-479 22 22 22 31 31 31 T T T 24 480-499 21 21 21 29 29 29 F F F 25 500-519 22 22 22 34 34 34 F F F 26 520-539 18 18 18 26 26 26 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 22 22 22 34 34 34 F F F 29 580-599 18 18 18 26 26 26 F F F Max gen: 22, Max flush: 34 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 34 34 34 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 21 21 21 29 29 29 F F F 3 060-079 21 21 21 29 29 29 F F F 4 080-099 14 14 14 21 21 21 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 19 19 19 27 27 27 F F F 7 140-159 21 21 21 29 29 29 F F F 8 160-179 22 22 22 34 34 34 F F F 9 180-199 23 23 23 35 35 35 F F F 10 200-219 22 22 22 34 34 34 F F F 11 220-239 21 21 21 29 29 29 F F F 12 240-259 22 22 22 34 34 34 F F F 13 260-279 23 23 23 35 35 35 F F F 14 280-299 21 21 21 29 29 29 F F F 15 300-319 21 21 21 29 29 29 F F F 16 320-339 18 18 18 26 26 26 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 16 16 16 24 24 24 F F F 19 380-399 23 23 23 35 35 35 F F F 20 400-419 23 23 23 35 35 35 F F F 21 420-439 18 18 18 26 26 26 F F F 22 440-459 23 23 23 35 35 35 F F F 23 460-479 23 23 23 35 35 35 F F F 24 480-499 21 21 21 29 29 29 F F F 25 500-519 22 22 22 34 34 34 F F F 26 520-539 18 18 18 26 26 26 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 22 22 22 34 34 34 F F F 29 580-599 18 18 18 26 26 26 F F F Max gen: 23, Max flush: 35 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.570853722Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.614707404Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.615443183Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.615494184Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.615511928Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.615531332Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284} {"msg":"Crucible 419d5c6d-197c-41fc-a58f-43360145d8b5 has session id: 9957719c-b901-4c5a-9853-e5b5b91617f8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616065737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Upstairs opts: Upstairs UUID: 419d5c6d-197c-41fc-a58f-43360145d8b5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616101625Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Crucible stats registered with UUID: 419d5c6d-197c-41fc-a58f-43360145d8b5","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616117508Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:23.616130801Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616601363Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616626088Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284} {"msg":"419d5c6d-197c-41fc-a58f-43360145d8b5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616672178Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616705285Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"0","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616744984Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"0","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616766668Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"1","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616788573Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"1","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616808608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"2","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616830202Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"2","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616849986Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"io task","client":"2","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616871061Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"io task","client":"0","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616894846Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"io task","client":"1","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49816)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616917051Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"io task","client":"0","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:43455)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.616939206Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"io task","client":"1","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:23.616963671Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"io task","client":"2","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:23.616987456Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"io task","client":"2","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:23.617014502Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.617243241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"io task","client":"2","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.617784738Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"0","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.617812254Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"1","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.618447061Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"0","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.618559455Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"1","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.61895288Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:23.619040969Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.618472666Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"io task","client":"2","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:50545)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.618547452Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"io task","client":"2","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.619698141Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"2","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.620263453Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"2","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"[0]R flush_numbers[0..12]: [34, 26, 29, 29, 21, 15, 27, 29, 34, 35, 34, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.62089813Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"[0]R generation[0..12]: [22, 18, 21, 21, 14, 10, 19, 21, 22, 23, 22, 21]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.620932167Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.620957052Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"[1]R flush_numbers[0..12]: [34, 26, 29, 29, 21, 15, 27, 29, 34, 35, 34, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.620985618Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"[1]R generation[0..12]: [22, 18, 21, 21, 14, 10, 19, 21, 22, 23, 22, 21]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.62103824Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621072097Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"[2]R flush_numbers[0..12]: [34, 26, 29, 29, 21, 15, 27, 29, 34, 35, 34, 29]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621091311Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"[2]R generation[0..12]: [22, 18, 21, 21, 14, 10, 19, 21, 22, 23, 22, 21]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621113286Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621145593Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.6211767Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621205106Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621231102Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"All extents match","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.62126859Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"":"downstairs","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621299826Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621324251Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"0","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621341875Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"1","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621368581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"client":"2","":"downstairs","session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621399398Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621425724Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"419d5c6d-197c-41fc-a58f-43360145d8b5 is now active with session: 9957719c-b901-4c5a-9853-e5b5b91617f8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621442367Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621466272Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284,"session_id":"9957719c-b901-4c5a-9853-e5b5b91617f8"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621493898Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284} {"msg":"Activated sub_volume 419d5c6d-197c-41fc-a58f-43360145d8b5","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:33.621519784Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 264 len 8192 data: 7 8 02/30 Read block 271 len 8192 03/30 Read block 567 len 32768 04/30 Write block 495 len 36864 data: 5 4 3 3 4 4 4 4 5 05/30 Read block 255 len 32768 06/30 Read block 481 len 24576 07/30 Read block 403 len 8192 08/30 Write block 526 len 16384 data: 6 7 6 5 09/30 Read block 356 len 32768 10/30 Write block 72 len 12288 data: 4 5 6 11/30 Read block 176 len 24576 12/30 Write block 298 len 16384 data: 4 3 3 3 13/30 Write block 443 len 20480 data: 4 3 3 3 2 14/30 Read block 36 len 16384 15/30 Flush 16/30 Write block 495 len 32768 data: 6 5 4 4 5 5 5 5 17/30 Read block 133 len 16384 18/30 Write block 136 len 4096 data: 3 19/30 Read block 177 len 12288 20/30 Write block 186 len 24576 data: 4 4 5 4 3 3 21/30 Write block 504 len 36864 data: 4 4 3 3 3 3 3 2 4 22/30 Read block 333 len 8192 23/30 Flush 24/30 Write block 52 len 12288 data: 7 6 5 25/30 Write block 181 len 40960 data: 3 3 3 4 4 5 5 6 5 4 26/30 Write block 217 len 32768 data: 4 4 5 5 5 4 4 3 27/30 Read block 162 len 16384 28/30 Read block 66 len 40960 29/30 Write block 472 len 24576 data: 3 3 3 3 3 3 30/30 Write block 177 len 36864 data: 5 4 5 5 4 4 4 5 5 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:33.641911964Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1284} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 34 34 34 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 21 21 21 29 29 29 T T T 3 060-079 24 24 24 36 36 36 F F F 4 080-099 14 14 14 21 21 21 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 21 21 21 29 29 29 F F F 8 160-179 22 22 22 34 34 34 T T T 9 180-199 24 24 24 37 37 37 T T T 10 200-219 22 22 22 34 34 34 T T T 11 220-239 21 21 21 29 29 29 T T T 12 240-259 22 22 22 34 34 34 F F F 13 260-279 24 24 24 36 36 36 F F F 14 280-299 24 24 24 36 36 36 F F F 15 300-319 24 24 24 36 36 36 F F F 16 320-339 18 18 18 26 26 26 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 16 16 16 24 24 24 F F F 19 380-399 23 23 23 35 35 35 F F F 20 400-419 23 23 23 35 35 35 F F F 21 420-439 18 18 18 26 26 26 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 23 23 23 35 35 35 T T T 24 480-499 24 24 24 37 37 37 F F F 25 500-519 24 24 24 37 37 37 F F F 26 520-539 24 24 24 36 36 36 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 22 22 22 34 34 34 F F F 29 580-599 18 18 18 26 26 26 F F F Max gen: 24, Max flush: 37 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 34 34 34 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 25 25 25 38 38 38 F F F 3 060-079 24 24 24 36 36 36 F F F 4 080-099 14 14 14 21 21 21 F F F 5 100-119 10 10 10 15 15 15 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 21 21 21 29 29 29 F F F 8 160-179 25 25 25 38 38 38 F F F 9 180-199 25 25 25 38 38 38 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 25 25 25 38 38 38 F F F 12 240-259 22 22 22 34 34 34 F F F 13 260-279 24 24 24 36 36 36 F F F 14 280-299 24 24 24 36 36 36 F F F 15 300-319 24 24 24 36 36 36 F F F 16 320-339 18 18 18 26 26 26 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 16 16 16 24 24 24 F F F 19 380-399 23 23 23 35 35 35 F F F 20 400-419 23 23 23 35 35 35 F F F 21 420-439 18 18 18 26 26 26 F F F 22 440-459 24 24 24 36 36 36 F F F 23 460-479 25 25 25 38 38 38 F F F 24 480-499 24 24 24 37 37 37 F F F 25 500-519 24 24 24 37 37 37 F F F 26 520-539 24 24 24 36 36 36 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 22 22 22 34 34 34 F F F 29 580-599 18 18 18 26 26 26 F F F Max gen: 25, Max flush: 38 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.889808974Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.934483283Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.935174953Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.935218282Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.935232865Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.935247148Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292} {"msg":"Crucible 27854046-b7b2-4e7f-8f1b-fcf615f2ab4c has session id: eb0141ee-1a86-4910-b008-06c64a13953e","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.935746486Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Upstairs opts: Upstairs UUID: 27854046-b7b2-4e7f-8f1b-fcf615f2ab4c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.935789805Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Crucible stats registered with UUID: 27854046-b7b2-4e7f-8f1b-fcf615f2ab4c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.93581288Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:37.935830624Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936389405Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936418801Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292} {"msg":"27854046-b7b2-4e7f-8f1b-fcf615f2ab4c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936472893Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.93650636Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"0","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936545028Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"0","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936567773Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"1","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936591828Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"1","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936614383Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"2","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936697221Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"2","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936725247Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"io task","client":"2","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936749172Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"io task","client":"0","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936771527Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"io task","client":"1","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45439)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936793692Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"io task","client":"2","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41864)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936816147Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"io task","client":"0","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54979)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.936846023Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"io task","client":"1","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.937389661Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"0","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.937418137Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"2","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.938024507Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"0","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.938168939Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"2","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.938658974Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.938787012Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.93914932Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"1","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.939627003Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"1","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"[0]R flush_numbers[0..12]: [34, 26, 38, 36, 21, 15, 37, 29, 38, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940138094Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"[0]R generation[0..12]: [22, 18, 25, 24, 14, 10, 24, 21, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940157608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940172011Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"[1]R flush_numbers[0..12]: [34, 26, 38, 36, 21, 15, 37, 29, 38, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940185534Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"[1]R generation[0..12]: [22, 18, 25, 24, 14, 10, 24, 21, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940198887Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.94021271Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"[2]R flush_numbers[0..12]: [34, 26, 38, 36, 21, 15, 37, 29, 38, 38, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940229833Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"[2]R generation[0..12]: [22, 18, 25, 24, 14, 10, 24, 21, 25, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940243676Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940257339Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940269762Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940282605Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940295097Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940317982Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"":"downstairs","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940334926Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940346768Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"0","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940359521Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"1","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940372024Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"client":"2","":"downstairs","session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940384487Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940396369Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"27854046-b7b2-4e7f-8f1b-fcf615f2ab4c is now active with session: eb0141ee-1a86-4910-b008-06c64a13953e","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940408582Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940420104Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292,"session_id":"eb0141ee-1a86-4910-b008-06c64a13953e"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.940431777Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292} {"msg":"Activated sub_volume 27854046-b7b2-4e7f-8f1b-fcf615f2ab4c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:37.94044446Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 54 len 8192 data: 6 5 02/30 Flush 03/30 Write block 372 len 16384 data: 3 3 3 3 04/30 Write block 165 len 40960 data: 9 6 3 3 3 3 2 3 4 4 05/30 Read block 94 len 12288 06/30 Write block 316 len 36864 data: 2 2 2 2 2 2 2 2 2 07/30 Read block 257 len 16384 08/30 Write block 230 len 4096 data: 2 09/30 Read block 247 len 16384 10/30 Read block 307 len 12288 11/30 Read block 161 len 20480 12/30 Write block 2 len 36864 data: 2 2 3 3 5 5 5 4 4 13/30 Read block 382 len 32768 14/30 Write block 275 len 4096 data: 3 15/30 Read block 149 len 16384 16/30 Write block 526 len 8192 data: 7 8 17/30 Write block 166 len 4096 data: 7 18/30 Read block 432 len 28672 19/30 Read block 115 len 40960 20/30 Read block 255 len 40960 21/30 Write block 583 len 40960 data: 2 4 4 3 3 3 4 4 4 3 22/30 Flush 23/30 Write block 224 len 20480 data: 4 2 2 2 2 24/30 Write block 404 len 28672 data: 2 2 2 3 4 4 4 25/30 Write block 518 len 28672 data: 6 4 4 3 5 6 5 26/30 Read block 182 len 16384 27/30 Write block 93 len 12288 data: 4 3 3 28/30 Write block 445 len 8192 data: 4 4 29/30 Read block 497 len 16384 30/30 Write block 97 len 32768 data: 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:37.960566591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1292} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 40 40 40 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 26 26 26 39 39 39 F F F 3 060-079 24 24 24 36 36 36 F F F 4 080-099 14 14 14 21 21 21 T T T 5 100-119 10 10 10 15 15 15 T T T 6 120-139 24 24 24 37 37 37 F F F 7 140-159 21 21 21 29 29 29 F F F 8 160-179 26 26 26 40 40 40 F F F 9 180-199 25 25 25 38 38 38 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 26 26 26 40 40 40 T T T 12 240-259 22 22 22 34 34 34 F F F 13 260-279 26 26 26 40 40 40 F F F 14 280-299 24 24 24 36 36 36 F F F 15 300-319 26 26 26 40 40 40 F F F 16 320-339 26 26 26 40 40 40 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 26 26 26 40 40 40 F F F 19 380-399 23 23 23 35 35 35 F F F 20 400-419 23 23 23 35 35 35 T T T 21 420-439 18 18 18 26 26 26 F F F 22 440-459 24 24 24 36 36 36 T T T 23 460-479 25 25 25 38 38 38 F F F 24 480-499 24 24 24 37 37 37 F F F 25 500-519 24 24 24 37 37 37 T T T 26 520-539 26 26 26 40 40 40 T T T 27 540-559 9 9 9 13 13 13 F F F 28 560-579 22 22 22 34 34 34 F F F 29 580-599 26 26 26 40 40 40 F F F Max gen: 26, Max flush: 40 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 40 40 40 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 26 26 26 39 39 39 F F F 3 060-079 24 24 24 36 36 36 F F F 4 080-099 27 27 27 41 41 41 F F F 5 100-119 27 27 27 41 41 41 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 21 21 21 29 29 29 F F F 8 160-179 26 26 26 40 40 40 F F F 9 180-199 25 25 25 38 38 38 F F F 10 200-219 25 25 25 38 38 38 F F F 11 220-239 27 27 27 41 41 41 F F F 12 240-259 22 22 22 34 34 34 F F F 13 260-279 26 26 26 40 40 40 F F F 14 280-299 24 24 24 36 36 36 F F F 15 300-319 26 26 26 40 40 40 F F F 16 320-339 26 26 26 40 40 40 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 26 26 26 40 40 40 F F F 19 380-399 23 23 23 35 35 35 F F F 20 400-419 27 27 27 41 41 41 F F F 21 420-439 18 18 18 26 26 26 F F F 22 440-459 27 27 27 41 41 41 F F F 23 460-479 25 25 25 38 38 38 F F F 24 480-499 24 24 24 37 37 37 F F F 25 500-519 27 27 27 41 41 41 F F F 26 520-539 27 27 27 41 41 41 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 22 22 22 34 34 34 F F F 29 580-599 26 26 26 40 40 40 F F F Max gen: 27, Max flush: 41 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.21379223Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.259710808Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.26041491Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.260465821Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.260484855Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.26050822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300} {"msg":"Crucible cb8281a4-a197-4159-ba6f-ecd986116c17 has session id: a02fa989-52c1-4a56-b3d0-b9a79110faf7","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261143918Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Upstairs opts: Upstairs UUID: cb8281a4-a197-4159-ba6f-ecd986116c17, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261186597Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Crucible stats registered with UUID: cb8281a4-a197-4159-ba6f-ecd986116c17","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261210152Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:42.261231006Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.26162042Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261647966Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300} {"msg":"cb8281a4-a197-4159-ba6f-ecd986116c17 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.26171132Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261735845Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"0","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261763591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"0","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261778454Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"1","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261792618Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"1","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261814392Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"2","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261833056Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"2","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261846209Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"io task","client":"2","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261869374Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"io task","client":"0","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.261888068Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"io task","client":"1","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:37373)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.262018706Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"io task","client":"0","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:63526)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.262068697Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"io task","client":"1","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"ds_connection connected from Ok(127.0.0.1:37332)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.262098624Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"io task","client":"2","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.262844304Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"0","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.262884463Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"2","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.263546156Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"2","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.263583684Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"0","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.264220961Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.264252108Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.264717979Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"1","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265199702Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"1","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"[0]R flush_numbers[0..12]: [40, 26, 39, 36, 41, 41, 37, 29, 40, 38, 38, 41]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265708432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"[0]R generation[0..12]: [26, 18, 26, 24, 27, 27, 24, 21, 26, 25, 25, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265741639Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265761194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"[1]R flush_numbers[0..12]: [40, 26, 39, 36, 41, 41, 37, 29, 40, 38, 38, 41]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265778778Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"[1]R generation[0..12]: [26, 18, 26, 24, 27, 27, 24, 21, 26, 25, 25, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265795801Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265813145Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"[2]R flush_numbers[0..12]: [40, 26, 39, 36, 41, 41, 37, 29, 40, 38, 38, 41]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265830109Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"[2]R generation[0..12]: [26, 18, 26, 24, 27, 27, 24, 21, 26, 25, 25, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265846892Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265863756Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Max found gen is 28","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265886341Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265902574Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265917988Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265933351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"":"downstairs","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265949124Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265963597Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"0","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.265979521Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"1","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.266015069Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"client":"2","":"downstairs","session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.266033342Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.266052057Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"cb8281a4-a197-4159-ba6f-ecd986116c17 is now active with session: a02fa989-52c1-4a56-b3d0-b9a79110faf7","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.266068Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.266090655Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300,"session_id":"a02fa989-52c1-4a56-b3d0-b9a79110faf7"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.266105928Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300} {"msg":"Activated sub_volume cb8281a4-a197-4159-ba6f-ecd986116c17","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:42.266119461Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 156 len 12288 data: 5 4 4 02/30 Read block 191 len 8192 03/30 Write block 281 len 24576 data: 3 3 3 4 4 4 04/30 Read block 6 len 16384 05/30 Read block 385 len 12288 06/30 Write block 439 len 32768 data: 4 4 5 5 5 4 5 5 07/30 Write block 407 len 4096 data: 4 08/30 Read block 205 len 20480 09/30 Read block 201 len 8192 10/30 Write block 311 len 32768 data: 3 3 3 3 3 3 3 3 11/30 Write block 484 len 40960 data: 3 3 3 4 4 7 6 6 7 8 12/30 Flush 13/30 Write block 173 len 12288 data: 5 5 4 14/30 Write block 360 len 24576 data: 5 4 5 5 4 4 15/30 Read block 541 len 24576 16/30 Write block 72 len 4096 data: 5 17/30 Write block 420 len 20480 data: 2 2 2 2 3 18/30 Flush 19/30 Write block 300 len 20480 data: 4 4 3 3 3 20/30 Read block 0 len 12288 21/30 Write block 178 len 32768 data: 5 6 6 5 5 5 6 6 22/30 Read block 310 len 28672 23/30 Flush 24/30 Write block 316 len 20480 data: 4 4 4 3 3 25/30 Write block 444 len 36864 data: 5 6 6 3 2 3 3 3 3 26/30 Read block 250 len 20480 27/30 Write block 153 len 36864 data: 2 4 5 6 5 5 5 5 6 28/30 Write block 203 len 24576 data: 2 2 2 3 3 3 29/30 Read block 99 len 36864 30/30 Write block 562 len 16384 data: 5 5 4 3 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:42.285839106Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1300} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 40 40 40 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 26 26 26 39 39 39 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 27 27 27 41 41 41 F F F 5 100-119 27 27 27 41 41 41 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 28 28 28 42 42 42 T T T 8 160-179 28 28 28 44 44 44 T T T 9 180-199 28 28 28 44 44 44 F F F 10 200-219 25 25 25 38 38 38 T T T 11 220-239 27 27 27 41 41 41 F F F 12 240-259 22 22 22 34 34 34 F F F 13 260-279 26 26 26 40 40 40 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 28 28 28 44 44 44 T T T 16 320-339 26 26 26 40 40 40 T T T 17 340-359 16 16 16 24 24 24 F F F 18 360-379 28 28 28 43 43 43 F F F 19 380-399 23 23 23 35 35 35 F F F 20 400-419 28 28 28 42 42 42 F F F 21 420-439 28 28 28 43 43 43 F F F 22 440-459 28 28 28 42 42 42 T T T 23 460-479 25 25 25 38 38 38 F F F 24 480-499 28 28 28 42 42 42 F F F 25 500-519 27 27 27 41 41 41 F F F 26 520-539 27 27 27 41 41 41 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 22 22 22 34 34 34 T T T 29 580-599 26 26 26 40 40 40 F F F Max gen: 28, Max flush: 44 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 40 40 40 F F F 1 020-039 18 18 18 26 26 26 F F F 2 040-059 26 26 26 39 39 39 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 27 27 27 41 41 41 F F F 5 100-119 27 27 27 41 41 41 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 29 29 29 45 45 45 F F F 8 160-179 29 29 29 45 45 45 F F F 9 180-199 28 28 28 44 44 44 F F F 10 200-219 29 29 29 45 45 45 F F F 11 220-239 27 27 27 41 41 41 F F F 12 240-259 22 22 22 34 34 34 F F F 13 260-279 26 26 26 40 40 40 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 29 29 29 45 45 45 F F F 16 320-339 29 29 29 45 45 45 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 28 28 28 43 43 43 F F F 19 380-399 23 23 23 35 35 35 F F F 20 400-419 28 28 28 42 42 42 F F F 21 420-439 28 28 28 43 43 43 F F F 22 440-459 29 29 29 45 45 45 F F F 23 460-479 25 25 25 38 38 38 F F F 24 480-499 28 28 28 42 42 42 F F F 25 500-519 27 27 27 41 41 41 F F F 26 520-539 27 27 27 41 41 41 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 29 29 29 45 45 45 F F F 29 580-599 26 26 26 40 40 40 F F F Max gen: 29, Max flush: 45 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.537390714Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.582917148Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.583706398Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.583748007Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.58376295Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.583777253Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308} {"msg":"Crucible e471e798-95aa-4c94-bfec-d66f18ab6b4a has session id: af8f7ac6-b186-44e4-a1bf-d40373777db3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.584289734Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Upstairs opts: Upstairs UUID: e471e798-95aa-4c94-bfec-d66f18ab6b4a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.584372632Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Crucible stats registered with UUID: e471e798-95aa-4c94-bfec-d66f18ab6b4a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.584405119Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.584443297Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.584930692Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.584964509Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308} {"msg":"e471e798-95aa-4c94-bfec-d66f18ab6b4a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.585020051Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.585049418Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"0","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.585082165Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"0","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.585103349Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"1","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.585118683Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"1","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.585131786Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"2","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.585194649Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"2","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.585220365Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"io task","client":"2","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.58524406Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"io task","client":"0","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.585271756Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"io task","client":"1","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"ds_connection connected from Ok(127.0.0.1:35052)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.585299892Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"io task","client":"0","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"ds_connection connected from Ok(127.0.0.1:54646)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.585330328Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"io task","client":"2","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"ds_connection connected from Ok(127.0.0.1:64497)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.585361375Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"io task","client":"1","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.586140933Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"2","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.58617327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"0","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.58631186Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"1","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.586833733Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"0","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.58686604Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"2","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.586882433Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"1","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587479882Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587505598Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"[0]R flush_numbers[0..12]: [40, 26, 39, 43, 41, 41, 37, 45, 45, 44, 45, 41]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587528003Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"[0]R generation[0..12]: [26, 18, 26, 28, 27, 27, 24, 29, 29, 28, 29, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587542716Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587556929Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"[1]R flush_numbers[0..12]: [40, 26, 39, 43, 41, 41, 37, 45, 45, 44, 45, 41]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587570241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"[1]R generation[0..12]: [26, 18, 26, 28, 27, 27, 24, 29, 29, 28, 29, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587583384Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587596727Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"[2]R flush_numbers[0..12]: [40, 26, 39, 43, 41, 41, 37, 45, 45, 44, 45, 41]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.58760997Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"[2]R generation[0..12]: [26, 18, 26, 28, 27, 27, 24, 29, 29, 28, 29, 27]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587623093Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587636756Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587649349Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587665472Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587678325Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587690858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"":"downstairs","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.58770357Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587714923Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"0","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587727726Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"1","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.58774796Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"client":"2","":"downstairs","session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587760873Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587772735Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"e471e798-95aa-4c94-bfec-d66f18ab6b4a is now active with session: af8f7ac6-b186-44e4-a1bf-d40373777db3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587785298Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587797041Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587808813Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308} {"msg":"Activated sub_volume e471e798-95aa-4c94-bfec-d66f18ab6b4a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:46.587819685Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 514 len 32768 data: 4 5 5 6 7 5 5 4 02/30 Write block 37 len 8192 data: 4 5 03/30 Read block 280 len 4096 04/30 Write block 467 len 40960 data: 3 3 3 3 2 4 4 4 4 4 05/30 Write block 403 len 20480 data: 2 3 3 3 5 06/30 Read block 432 len 36864 07/30 Write block 179 len 4096 data: 7 08/30 Write block 577 len 36864 data: 4 4 4 3 3 3 3 5 5 09/30 Flush 10/30 Write block 481 len 28672 data: 2 2 2 4 4 4 5 11/30 Write block 333 len 28672 data: 5 5 6 4 4 3 4 12/30 Read block 413 len 28672 13/30 Read block 163 len 24576 14/30 Write block 442 len 24576 data: 6 6 6 7 7 4 15/30 Read block 345 len 20480 16/30 Write block 486 len 24576 data: 5 6 5 8 7 7 17/30 Read block 320 len 36864 18/30 Read block 63 len 32768 19/30 Write block 227 len 16384 data: 3 3 2 3 20/30 Write block 139 len 28672 data: 4 2 2 3 3 3 3 21/30 Flush 22/30 Write block 180 len 4096 data: 7 23/30 Write block 224 len 32768 data: 5 3 3 4 4 3 4 2 24/30 Read block 209 len 20480 25/30 Write block 259 len 36864 data: 4 4 4 4 6 8 9 7 6 26/30 Flush 27/30 Write block 104 len 16384 data: 3 2 2 2 28/30 Write block 116 len 32768 data: 4 4 4 4 4 4 6 5 29/30 Write block 376 len 32768 data: 3 2 3 3 3 3 2 3 30/30 Write block 315 len 24576 data: 4 5 5 5 4 4 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.608813795Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308} Wrote out file "/tmp/test_repair_verify.data" {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609478029Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609514467Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609541102Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609563777Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609586222Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609610207Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609632412Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609651786Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.60966926Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609689504Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609706588Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609719801Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609732494Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609746606Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609765321Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609785945Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609803129Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609816842Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609828804Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609840527Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609852149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609864522Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609876225Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609887837Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.60990021Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609912632Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609924395Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.60994609Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609967724Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.609980927Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.60999263Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610005482Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610017705Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610029778Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.61004162Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610053423Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610064985Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610076428Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.61008857Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610101063Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610112856Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610125268Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610137151Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610148753Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610163416Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610175559Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610187852Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610199454Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610211097Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610222739Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610235272Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610247154Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610258797Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.61027048Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610282782Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610294445Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610305937Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.61031761Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610329943Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610341675Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610353548Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610367531Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610380423Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610392306Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610404929Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610417051Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610429704Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610441427Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610453199Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610465762Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610477774Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610489457Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.6105019Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610513932Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610525755Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610537277Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.61054874Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610560962Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610574845Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610587418Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.61059913Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610610533Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610622446Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610634318Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610646081Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610658023Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610669846Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610681668Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610693581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610705373Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610717316Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610729038Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610740841Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610752494Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610764296Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610778419Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610791402Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610803425Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610815197Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.61082704Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610839562Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610851425Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610863027Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.61087527Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610887673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610900066Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610912388Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610924081Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610935703Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610947186Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610958648Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.61097019Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610984644Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.610997327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.611009169Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.611020612Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.611032204Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.611043697Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.611055389Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.611066971Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.611078404Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.611089796Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.611101449Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:46.611112911Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1308,"session_id":"af8f7ac6-b186-44e4-a1bf-d40373777db3"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 40 40 40 F F F 1 020-039 30 30 30 46 46 46 F F F 2 040-059 26 26 26 39 39 39 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 27 27 27 41 41 41 F F F 5 100-119 27 27 27 41 41 41 T T T 6 120-139 30 30 30 47 47 47 T T T 7 140-159 30 30 30 47 47 47 F F F 8 160-179 30 30 30 46 46 46 F F F 9 180-199 30 30 30 48 48 48 F F F 10 200-219 29 29 29 45 45 45 F F F 11 220-239 30 30 30 48 48 48 F F F 12 240-259 30 30 30 48 48 48 F F F 13 260-279 30 30 30 48 48 48 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 29 29 29 45 45 45 T T T 16 320-339 30 30 30 47 47 47 T T T 17 340-359 16 16 16 24 24 24 F F F 18 360-379 28 28 28 43 43 43 T T T 19 380-399 23 23 23 35 35 35 T T T 20 400-419 30 30 30 46 46 46 F F F 21 420-439 28 28 28 43 43 43 F F F 22 440-459 30 30 30 47 47 47 F F F 23 460-479 30 30 30 46 46 46 F F F 24 480-499 30 30 30 47 47 47 F F F 25 500-519 30 30 30 46 46 46 F F F 26 520-539 30 30 30 46 46 46 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 30 30 30 46 46 46 F F F 29 580-599 30 30 30 46 46 46 F F F Max gen: 30, Max flush: 48 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 40 40 40 F F F 1 020-039 30 30 30 46 46 46 F F F 2 040-059 26 26 26 39 39 39 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 27 27 27 41 41 41 F F F 5 100-119 31 31 31 49 49 49 F F F 6 120-139 31 31 31 49 49 49 F F F 7 140-159 30 30 30 47 47 47 F F F 8 160-179 30 30 30 46 46 46 F F F 9 180-199 30 30 30 48 48 48 F F F 10 200-219 29 29 29 45 45 45 F F F 11 220-239 30 30 30 48 48 48 F F F 12 240-259 30 30 30 48 48 48 F F F 13 260-279 30 30 30 48 48 48 F F F 14 280-299 28 28 28 42 42 42 F F F 15 300-319 31 31 31 49 49 49 F F F 16 320-339 31 31 31 49 49 49 F F F 17 340-359 16 16 16 24 24 24 F F F 18 360-379 31 31 31 49 49 49 F F F 19 380-399 31 31 31 49 49 49 F F F 20 400-419 30 30 30 46 46 46 F F F 21 420-439 28 28 28 43 43 43 F F F 22 440-459 30 30 30 47 47 47 F F F 23 460-479 30 30 30 46 46 46 F F F 24 480-499 30 30 30 47 47 47 F F F 25 500-519 30 30 30 46 46 46 F F F 26 520-539 30 30 30 46 46 46 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 30 30 30 46 46 46 F F F 29 580-599 30 30 30 46 46 46 F F F Max gen: 31, Max flush: 49 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.855602975Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.899839951Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.900666879Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.900709898Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.900724601Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.900739314Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316} {"msg":"Crucible d28299fe-87b7-4a9c-9e05-48e7817de7a1 has session id: f995db4e-14aa-48e5-be1b-4008d949d537","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.901441596Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Upstairs opts: Upstairs UUID: d28299fe-87b7-4a9c-9e05-48e7817de7a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.901488746Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Crucible stats registered with UUID: d28299fe-87b7-4a9c-9e05-48e7817de7a1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.901514462Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:50.901535266Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.90206327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902089376Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316} {"msg":"d28299fe-87b7-4a9c-9e05-48e7817de7a1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902163152Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902180876Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"0","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902209002Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"0","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902222765Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"1","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902236908Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"1","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902249831Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"2","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902263293Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"2","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902276026Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.90229297Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902344551Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"ds_connection connected from Ok(127.0.0.1:49852)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902380179Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"ds_connection connected from Ok(127.0.0.1:36603)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902597566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"ds_connection connected from Ok(127.0.0.1:61714)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.902638894Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.903159067Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"2","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.903186542Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"0","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.903699673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"0","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.903846505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"2","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.904325408Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.904418868Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"1","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.904529172Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.90507533Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"1","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"[0]R flush_numbers[0..12]: [40, 46, 39, 43, 41, 49, 49, 47, 46, 48, 45, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905603684Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"[0]R generation[0..12]: [26, 30, 26, 28, 27, 31, 31, 30, 30, 30, 29, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905625629Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905643673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"[1]R flush_numbers[0..12]: [40, 46, 39, 43, 41, 49, 49, 47, 46, 48, 45, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905658486Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"[1]R generation[0..12]: [26, 30, 26, 28, 27, 31, 31, 30, 30, 30, 29, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905671949Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905685472Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"[2]R flush_numbers[0..12]: [40, 46, 39, 43, 41, 49, 49, 47, 46, 48, 45, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905698555Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"[2]R generation[0..12]: [26, 30, 26, 28, 27, 31, 31, 30, 30, 30, 29, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905711487Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.90572494Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905738443Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905751236Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905763719Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:20:50.905776191Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"":"downstairs","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905793315Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905804947Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"0","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905818461Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"1","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905831083Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"client":"2","":"downstairs","session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905846636Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905858869Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"d28299fe-87b7-4a9c-9e05-48e7817de7a1 is now active with session: f995db4e-14aa-48e5-be1b-4008d949d537","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905871342Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905883404Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316,"session_id":"f995db4e-14aa-48e5-be1b-4008d949d537"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905895187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316} {"msg":"Activated sub_volume d28299fe-87b7-4a9c-9e05-48e7817de7a1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:20:50.905906099Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 509 len 36864 data: 4 4 3 5 4 5 6 6 7 02/30 Read block 561 len 32768 03/30 Write block 440 len 20480 data: 5 6 7 7 7 04/30 Write block 501 len 4096 data: 6 05/30 Read block 241 len 28672 06/30 Read block 264 len 32768 07/30 Read block 527 len 40960 08/30 Write block 426 len 32768 data: 3 4 5 4 4 3 3 4 09/30 Write block 483 len 24576 data: 3 5 5 6 7 6 10/30 Write block 338 len 12288 data: 4 5 4 11/30 Flush 12/30 Write block 267 len 40960 data: 7 6 5 4 3 4 4 3 4 3 13/30 Write block 569 len 32768 data: 4 4 4 4 4 5 6 4 14/30 Read block 60 len 16384 15/30 Read block 429 len 36864 16/30 Read block 341 len 16384 17/30 Write block 492 len 32768 data: 8 9 7 7 6 5 5 6 18/30 Read block 252 len 28672 19/30 Write block 430 len 28672 data: 5 4 4 5 4 4 4 20/30 Read block 90 len 40960 21/30 Write block 285 len 28672 data: 5 5 3 3 3 4 2 22/30 Write block 497 len 28672 data: 6 6 7 6 7 6 6 23/30 Write block 213 len 8192 data: 2 2 24/30 Read block 128 len 40960 25/30 Read block 2 len 24576 26/30 Read block 432 len 40960 27/30 Read block 354 len 16384 28/30 Read block 348 len 28672 29/30 Read block 481 len 8192 30/30 Write block 572 len 40960 data: 5 5 6 7 5 5 5 5 4 4 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 10 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:20:50.92787725Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1316} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 40 40 40 F F F 1 020-039 30 30 30 46 46 46 F F F 2 040-059 26 26 26 39 39 39 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 27 27 27 41 41 41 F F F 5 100-119 31 31 31 49 49 49 F F F 6 120-139 31 31 31 49 49 49 F F F 7 140-159 30 30 30 47 47 47 F F F 8 160-179 30 30 30 46 46 46 F F F 9 180-199 30 30 30 48 48 48 F F F 10 200-219 29 29 29 45 45 45 T T T 11 220-239 30 30 30 48 48 48 F F F 12 240-259 30 30 30 48 48 48 F F F 13 260-279 30 30 30 48 48 48 T T T 14 280-299 28 28 28 42 42 42 T T T 15 300-319 31 31 31 49 49 49 F F F 16 320-339 32 32 32 50 50 50 F F F 17 340-359 32 32 32 50 50 50 F F F 18 360-379 31 31 31 49 49 49 F F F 19 380-399 31 31 31 49 49 49 F F F 20 400-419 30 30 30 46 46 46 F F F 21 420-439 32 32 32 50 50 50 T T T 22 440-459 32 32 32 50 50 50 F F F 23 460-479 30 30 30 46 46 46 F F F 24 480-499 32 32 32 50 50 50 T T T 25 500-519 32 32 32 50 50 50 T T T 26 520-539 30 30 30 46 46 46 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 30 30 30 46 46 46 T T T 29 580-599 30 30 30 46 46 46 T T T Max gen: 32, Max flush: 50 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 40 40 40 F F F 1 020-039 30 30 30 46 46 46 F F F 2 040-059 26 26 26 39 39 39 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 27 27 27 41 41 41 F F F 5 100-119 31 31 31 49 49 49 F F F 6 120-139 31 31 31 49 49 49 F F F 7 140-159 30 30 30 47 47 47 F F F 8 160-179 30 30 30 46 46 46 F F F 9 180-199 30 30 30 48 48 48 F F F 10 200-219 33 33 33 51 51 51 F F F 11 220-239 30 30 30 48 48 48 F F F 12 240-259 30 30 30 48 48 48 F F F 13 260-279 33 33 33 51 51 51 F F F 14 280-299 33 33 33 51 51 51 F F F 15 300-319 31 31 31 49 49 49 F F F 16 320-339 32 32 32 50 50 50 F F F 17 340-359 32 32 32 50 50 50 F F F 18 360-379 31 31 31 49 49 49 F F F 19 380-399 31 31 31 49 49 49 F F F 20 400-419 30 30 30 46 46 46 F F F 21 420-439 33 33 33 51 51 51 F F F 22 440-459 32 32 32 50 50 50 F F F 23 460-479 30 30 30 46 46 46 F F F 24 480-499 33 33 33 51 51 51 F F F 25 500-519 33 33 33 51 51 51 F F F 26 520-539 30 30 30 46 46 46 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 33 33 33 51 51 51 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 33, Max flush: 51 Begin loop 17 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.18592318Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:05.228380651Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.228900023Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.260395709Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.261011032Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.261046319Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.261060182Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.261074456Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324} {"msg":"Crucible 04585a54-44c1-4ba6-8b00-484a490d0ee5 has session id: b511fc1e-990d-4b5b-8a06-49da34ec75a6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.26178998Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Upstairs opts: Upstairs UUID: 04585a54-44c1-4ba6-8b00-484a490d0ee5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.26183606Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Crucible stats registered with UUID: 04585a54-44c1-4ba6-8b00-484a490d0ee5","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.261856184Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:05.261870067Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262315013Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262353441Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324} {"msg":"04585a54-44c1-4ba6-8b00-484a490d0ee5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262407943Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262432409Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"0","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262470427Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"0","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262498243Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"1","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262527909Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"1","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262556835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"2","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262583891Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"2","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262598825Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"io task","client":"2","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262620819Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"io task","client":"0","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262649195Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"io task","client":"1","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61256)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262679112Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"io task","client":"2","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:47020)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262707448Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"io task","client":"1","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:38483)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.262736664Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"io task","client":"0","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.263488476Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"1","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.263520573Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"2","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.263554921Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"0","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.264182946Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"1","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.264343351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"0","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.264375918Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"2","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.264742057Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.264910723Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"[0]R flush_numbers[0..12]: [40, 46, 39, 43, 41, 49, 49, 47, 46, 48, 51, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265049453Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"[0]R generation[0..12]: [26, 30, 26, 28, 27, 31, 31, 30, 30, 30, 33, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265077119Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265092632Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"[1]R flush_numbers[0..12]: [40, 46, 39, 43, 41, 49, 49, 47, 46, 48, 51, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265107116Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"[1]R generation[0..12]: [26, 30, 26, 28, 27, 31, 31, 30, 30, 30, 33, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265120709Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265134492Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"[2]R flush_numbers[0..12]: [40, 46, 39, 43, 41, 49, 49, 47, 46, 48, 51, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265147835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"[2]R generation[0..12]: [26, 30, 26, 28, 27, 31, 31, 30, 30, 30, 33, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265165508Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265179811Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265192984Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265206057Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Wait for a query_work_queue command to finish before sending IO Next flush: 52","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265225941Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265238874Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"":"downstairs","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265251997Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265263769Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"0","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265276842Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"1","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265289685Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"client":"2","":"downstairs","session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265302428Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.26531431Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"04585a54-44c1-4ba6-8b00-484a490d0ee5 is now active with session: b511fc1e-990d-4b5b-8a06-49da34ec75a6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265326763Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265338336Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324,"session_id":"b511fc1e-990d-4b5b-8a06-49da34ec75a6"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.265350238Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324} {"msg":"Activated sub_volume 04585a54-44c1-4ba6-8b00-484a490d0ee5","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:05.26536115Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 467 len 36864 data: 4 4 4 4 3 5 5 5 5 02/30 Read block 235 len 12288 03/30 Read block 443 len 40960 04/30 Write block 400 len 8192 data: 3 2 05/30 Write block 503 len 16384 data: 7 5 5 4 06/30 Write block 407 len 12288 data: 6 5 5 07/30 Write block 20 len 12288 data: 3 3 4 08/30 Read block 45 len 16384 09/30 Write block 500 len 20480 data: 7 8 7 8 6 10/30 Read block 129 len 40960 11/30 Write block 37 len 8192 data: 5 6 12/30 Write block 331 len 32768 data: 6 6 6 6 7 5 5 5 13/30 Read block 386 len 12288 14/30 Write block 501 len 40960 data: 9 8 9 7 6 5 4 4 5 5 15/30 Write block 567 len 8192 data: 5 4 16/30 Flush 17/30 Write block 360 len 16384 data: 6 5 6 6 18/30 Read block 585 len 16384 19/30 Flush 20/30 Write block 263 len 40960 data: 7 9 10 8 8 7 6 5 4 5 21/30 Write block 26 len 20480 data: 2 2 2 2 2 22/30 Flush 23/30 Write block 226 len 40960 data: 4 5 5 4 5 3 2 3 4 4 24/30 Read block 557 len 28672 25/30 Write block 460 len 36864 data: 3 4 4 4 4 4 2 5 5 26/30 Write block 48 len 16384 data: 7 6 6 5 27/30 Write block 223 len 32768 data: 5 6 4 5 6 6 5 6 28/30 Read block 415 len 20480 29/30 Write block 396 len 32768 data: 4 4 4 4 4 3 2 3 30/30 Read block 131 len 8192 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:05.286381096Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1324} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 40 40 40 F F F 1 020-039 34 34 34 54 54 54 F F F 2 040-059 26 26 26 39 39 39 T T T 3 060-079 28 28 28 43 43 43 F F F 4 080-099 27 27 27 41 41 41 F F F 5 100-119 31 31 31 49 49 49 F F F 6 120-139 31 31 31 49 49 49 F F F 7 140-159 30 30 30 47 47 47 F F F 8 160-179 30 30 30 46 46 46 F F F 9 180-199 30 30 30 48 48 48 F F F 10 200-219 33 33 33 51 51 51 F F F 11 220-239 30 30 30 48 48 48 T T T 12 240-259 30 30 30 48 48 48 F F F 13 260-279 34 34 34 54 54 54 F F F 14 280-299 33 33 33 51 51 51 F F F 15 300-319 31 31 31 49 49 49 F F F 16 320-339 34 34 34 52 52 52 F F F 17 340-359 32 32 32 50 50 50 F F F 18 360-379 34 34 34 53 53 53 F F F 19 380-399 31 31 31 49 49 49 T T T 20 400-419 34 34 34 52 52 52 T T T 21 420-439 33 33 33 51 51 51 F F F 22 440-459 32 32 32 50 50 50 F F F 23 460-479 34 34 34 52 52 52 T T T 24 480-499 33 33 33 51 51 51 F F F 25 500-519 34 34 34 52 52 52 F F F 26 520-539 30 30 30 46 46 46 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 34 34 34 52 52 52 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 34, Max flush: 54 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 40 40 40 F F F 1 020-039 34 34 34 54 54 54 F F F 2 040-059 35 35 35 55 55 55 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 27 27 27 41 41 41 F F F 5 100-119 31 31 31 49 49 49 F F F 6 120-139 31 31 31 49 49 49 F F F 7 140-159 30 30 30 47 47 47 F F F 8 160-179 30 30 30 46 46 46 F F F 9 180-199 30 30 30 48 48 48 F F F 10 200-219 33 33 33 51 51 51 F F F 11 220-239 35 35 35 55 55 55 F F F 12 240-259 30 30 30 48 48 48 F F F 13 260-279 34 34 34 54 54 54 F F F 14 280-299 33 33 33 51 51 51 F F F 15 300-319 31 31 31 49 49 49 F F F 16 320-339 34 34 34 52 52 52 F F F 17 340-359 32 32 32 50 50 50 F F F 18 360-379 34 34 34 53 53 53 F F F 19 380-399 35 35 35 55 55 55 F F F 20 400-419 35 35 35 55 55 55 F F F 21 420-439 33 33 33 51 51 51 F F F 22 440-459 32 32 32 50 50 50 F F F 23 460-479 35 35 35 55 55 55 F F F 24 480-499 33 33 33 51 51 51 F F F 25 500-519 34 34 34 52 52 52 F F F 26 520-539 30 30 30 46 46 46 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 34 34 34 52 52 52 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 35, Max flush: 55 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.558844256Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.603830883Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.604595248Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.60465085Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.604669194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.604687348Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332} {"msg":"Crucible c1ecdcfe-f6c7-4622-9078-74f2bd122abd has session id: a37fd2f9-a5cf-4d24-9a70-5591af3de3e9","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.605325276Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Upstairs opts: Upstairs UUID: c1ecdcfe-f6c7-4622-9078-74f2bd122abd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.605371926Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Crucible stats registered with UUID: c1ecdcfe-f6c7-4622-9078-74f2bd122abd","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.605398801Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:09.605420746Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.605836456Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.605858711Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332} {"msg":"c1ecdcfe-f6c7-4622-9078-74f2bd122abd active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.605925735Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.605942419Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"0","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.605970275Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"0","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.605984038Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"1","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.60604046Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"1","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.606069406Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"2","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.606097862Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"2","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.606122177Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"io task","client":"2","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.606154595Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"io task","client":"0","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.60618026Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"io task","client":"1","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54694)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.606205095Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"io task","client":"2","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:42779)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.606231961Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"io task","client":"0","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:46164)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.606263198Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"io task","client":"1","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.606984123Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"1","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.607024042Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"0","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.607172104Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"2","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.607608228Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"1","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.607636814Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"0","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.607652358Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"2","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608198765Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608300358Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"[0]R flush_numbers[0..12]: [40, 54, 55, 43, 41, 49, 49, 47, 46, 48, 51, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608386196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"[0]R generation[0..12]: [26, 34, 35, 28, 27, 31, 31, 30, 30, 30, 33, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.60840658Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608424624Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"[1]R flush_numbers[0..12]: [40, 54, 55, 43, 41, 49, 49, 47, 46, 48, 51, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608443898Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"[1]R generation[0..12]: [26, 34, 35, 28, 27, 31, 31, 30, 30, 30, 33, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608467914Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608494519Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"[2]R flush_numbers[0..12]: [40, 54, 55, 43, 41, 49, 49, 47, 46, 48, 51, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608516224Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"[2]R generation[0..12]: [26, 34, 35, 28, 27, 31, 31, 30, 30, 30, 33, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608533638Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608558573Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608584389Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608607744Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608621787Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608640161Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"":"downstairs","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608669117Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608692412Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"0","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608713436Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"1","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.60873079Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"client":"2","":"downstairs","session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608748174Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608769629Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"c1ecdcfe-f6c7-4622-9078-74f2bd122abd is now active with session: a37fd2f9-a5cf-4d24-9a70-5591af3de3e9","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608793644Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608814578Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332,"session_id":"a37fd2f9-a5cf-4d24-9a70-5591af3de3e9"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608827191Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332} {"msg":"Activated sub_volume c1ecdcfe-f6c7-4622-9078-74f2bd122abd","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:09.608840314Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 203 len 28672 data: 3 3 3 4 4 4 3 02/30 Read block 481 len 28672 03/30 Read block 293 len 28672 04/30 Read block 552 len 36864 05/30 Read block 336 len 16384 06/30 Write block 418 len 40960 data: 2 2 3 3 3 3 4 3 4 5 07/30 Write block 558 len 16384 data: 4 4 5 5 08/30 Write block 1 len 4096 data: 2 09/30 Write block 0 len 20480 data: 2 3 3 3 4 10/30 Read block 112 len 28672 11/30 Read block 357 len 20480 12/30 Write block 437 len 4096 data: 4 13/30 Read block 374 len 8192 14/30 Read block 405 len 24576 15/30 Flush 16/30 Write block 371 len 20480 data: 2 4 4 4 4 17/30 Read block 346 len 16384 18/30 Read block 214 len 12288 19/30 Write block 534 len 16384 data: 2 2 2 2 20/30 Write block 200 len 28672 data: 3 3 2 4 4 4 5 21/30 Read block 471 len 8192 22/30 Read block 567 len 16384 23/30 Read block 269 len 16384 24/30 Read block 501 len 28672 25/30 Read block 47 len 32768 26/30 Write block 86 len 20480 data: 3 3 3 2 4 27/30 Write block 294 len 32768 data: 3 3 4 4 5 4 5 5 28/30 Write block 131 len 12288 data: 3 3 3 29/30 Read block 341 len 32768 30/30 Write block 405 len 12288 data: 4 4 7 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 5 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 14 14 13 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:09.628678164Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1332} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 56 56 56 F F F 1 020-039 34 34 34 54 54 54 F F F 2 040-059 35 35 35 55 55 55 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 27 27 27 41 41 41 T T T 5 100-119 31 31 31 49 49 49 F F F 6 120-139 31 31 31 49 49 49 T T T 7 140-159 30 30 30 47 47 47 F F F 8 160-179 30 30 30 46 46 46 F F F 9 180-199 30 30 30 48 48 48 F F F 10 200-219 36 36 36 56 56 56 T T T 11 220-239 35 35 35 55 55 55 F F F 12 240-259 30 30 30 48 48 48 F F F 13 260-279 34 34 34 54 54 54 F F F 14 280-299 33 33 33 51 51 51 T T T 15 300-319 31 31 31 49 49 49 T T T 16 320-339 34 34 34 52 52 52 F F F 17 340-359 32 32 32 50 50 50 F F F 18 360-379 34 34 34 53 53 53 T T T 19 380-399 35 35 35 55 55 55 F F F 20 400-419 36 36 36 56 56 56 T T T 21 420-439 36 36 36 56 56 56 F F F 22 440-459 32 32 32 50 50 50 F F F 23 460-479 35 35 35 55 55 55 F F F 24 480-499 33 33 33 51 51 51 F F F 25 500-519 34 34 34 52 52 52 F F F 26 520-539 30 30 30 46 46 46 T T T 27 540-559 36 36 36 56 56 56 F F F 28 560-579 36 36 36 56 56 56 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 36, Max flush: 56 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 56 56 56 F F F 1 020-039 34 34 34 54 54 54 F F F 2 040-059 35 35 35 55 55 55 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 37 37 37 57 57 57 F F F 5 100-119 31 31 31 49 49 49 F F F 6 120-139 37 37 37 57 57 57 F F F 7 140-159 30 30 30 47 47 47 F F F 8 160-179 30 30 30 46 46 46 F F F 9 180-199 30 30 30 48 48 48 F F F 10 200-219 37 37 37 57 57 57 F F F 11 220-239 35 35 35 55 55 55 F F F 12 240-259 30 30 30 48 48 48 F F F 13 260-279 34 34 34 54 54 54 F F F 14 280-299 37 37 37 57 57 57 F F F 15 300-319 37 37 37 57 57 57 F F F 16 320-339 34 34 34 52 52 52 F F F 17 340-359 32 32 32 50 50 50 F F F 18 360-379 37 37 37 57 57 57 F F F 19 380-399 35 35 35 55 55 55 F F F 20 400-419 37 37 37 57 57 57 F F F 21 420-439 36 36 36 56 56 56 F F F 22 440-459 32 32 32 50 50 50 F F F 23 460-479 35 35 35 55 55 55 F F F 24 480-499 33 33 33 51 51 51 F F F 25 500-519 34 34 34 52 52 52 F F F 26 520-539 37 37 37 57 57 57 F F F 27 540-559 36 36 36 56 56 56 F F F 28 560-579 36 36 36 56 56 56 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 37, Max flush: 57 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.881910855Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:13.924540983Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.924956333Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.956036109Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.956749253Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.95678421Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.956798023Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.956812766Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340} {"msg":"Crucible cc725e39-1f19-4c22-8f2d-0a34d0962768 has session id: 106624e2-d9dc-459b-9cc4-b2b63fecb949","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.957409915Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Upstairs opts: Upstairs UUID: cc725e39-1f19-4c22-8f2d-0a34d0962768, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.957455155Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Crucible stats registered with UUID: cc725e39-1f19-4c22-8f2d-0a34d0962768","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.95747757Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:13.957491383Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.957866164Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.957891519Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340} {"msg":"cc725e39-1f19-4c22-8f2d-0a34d0962768 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.957949502Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.957968226Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"0","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.958000333Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"0","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.958026988Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"1","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.958054504Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"1","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.95807997Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"2","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.958109826Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"2","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.958136042Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"io task","client":"2","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.958167339Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"io task","client":"0","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.958197985Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"io task","client":"1","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"ds_connection connected from Ok(127.0.0.1:34210)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.958227892Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"io task","client":"2","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"ds_connection connected from Ok(127.0.0.1:59192)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.958258758Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"io task","client":"1","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"ds_connection connected from Ok(127.0.0.1:37651)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.958285904Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"io task","client":"0","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.958918841Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"1","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.958949918Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"2","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.959500957Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"0","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.959579803Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"2","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.959653029Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"1","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960178052Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960203448Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960217881Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"0","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"[0]R flush_numbers[0..12]: [56, 54, 55, 43, 57, 49, 57, 47, 46, 48, 57, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960760528Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"[0]R generation[0..12]: [36, 34, 35, 28, 37, 31, 37, 30, 30, 30, 37, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960785484Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960800457Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"[1]R flush_numbers[0..12]: [56, 54, 55, 43, 57, 49, 57, 47, 46, 48, 57, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.96081386Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"[1]R generation[0..12]: [36, 34, 35, 28, 37, 31, 37, 30, 30, 30, 37, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960827393Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960840996Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"[2]R flush_numbers[0..12]: [56, 54, 55, 43, 57, 49, 57, 47, 46, 48, 57, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960854338Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"[2]R generation[0..12]: [36, 34, 35, 28, 37, 31, 37, 30, 30, 30, 37, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960867171Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960883785Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960896978Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.96091009Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960922673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-17T22:21:13.960935216Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"":"downstairs","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.96095428Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960965793Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"0","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960978445Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"1","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.960991098Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"client":"2","":"downstairs","session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.961003831Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.961015673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"cc725e39-1f19-4c22-8f2d-0a34d0962768 is now active with session: 106624e2-d9dc-459b-9cc4-b2b63fecb949","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.961028036Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.961039529Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340,"session_id":"106624e2-d9dc-459b-9cc4-b2b63fecb949"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.961052421Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340} {"msg":"Activated sub_volume cc725e39-1f19-4c22-8f2d-0a34d0962768","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:13.961063654Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 348 len 16384 data: 2 2 2 2 02/30 Flush 03/30 Write block 283 len 8192 data: 4 5 04/30 Flush 05/30 Write block 546 len 40960 data: 3 3 3 2 2 2 2 2 2 2 06/30 Read block 164 len 36864 07/30 Read block 518 len 40960 08/30 Read block 59 len 36864 09/30 Flush 10/30 Write block 438 len 12288 data: 3 5 6 11/30 Flush 12/30 Write block 577 len 12288 data: 6 6 6 13/30 Write block 323 len 24576 data: 3 3 3 3 3 3 14/30 Read block 389 len 20480 15/30 Write block 150 len 40960 data: 3 4 3 3 5 6 7 6 6 6 16/30 Write block 482 len 4096 data: 3 17/30 Read block 577 len 20480 18/30 Read block 432 len 28672 19/30 Read block 39 len 24576 20/30 Read block 311 len 24576 21/30 Write block 165 len 24576 data: 10 8 4 4 4 4 22/30 Read block 516 len 36864 23/30 Read block 4 len 8192 24/30 Write block 180 len 32768 data: 8 6 6 6 7 7 6 6 25/30 Flush 26/30 Write block 259 len 32768 data: 5 5 5 5 8 10 11 9 27/30 Read block 335 len 24576 28/30 Write block 20 len 4096 data: 4 29/30 Write block 333 len 40960 data: 7 7 8 6 6 6 6 5 3 3 30/30 Read block 49 len 36864 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 3 5 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:13.98218128Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1340} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 56 56 56 F F F 1 020-039 34 34 34 54 54 54 T T T 2 040-059 35 35 35 55 55 55 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 37 37 37 57 57 57 F F F 5 100-119 31 31 31 49 49 49 F F F 6 120-139 37 37 37 57 57 57 F F F 7 140-159 38 38 38 62 62 62 F F F 8 160-179 38 38 38 62 62 62 F F F 9 180-199 38 38 38 62 62 62 F F F 10 200-219 37 37 37 57 57 57 F F F 11 220-239 35 35 35 55 55 55 F F F 12 240-259 30 30 30 48 48 48 T T T 13 260-279 34 34 34 54 54 54 T T T 14 280-299 38 38 38 59 59 59 F F F 15 300-319 37 37 37 57 57 57 F F F 16 320-339 38 38 38 62 62 62 T T T 17 340-359 38 38 38 58 58 58 T T T 18 360-379 37 37 37 57 57 57 F F F 19 380-399 35 35 35 55 55 55 F F F 20 400-419 37 37 37 57 57 57 F F F 21 420-439 38 38 38 61 61 61 F F F 22 440-459 38 38 38 61 61 61 F F F 23 460-479 35 35 35 55 55 55 F F F 24 480-499 38 38 38 62 62 62 F F F 25 500-519 34 34 34 52 52 52 F F F 26 520-539 37 37 37 57 57 57 F F F 27 540-559 38 38 38 60 60 60 F F F 28 560-579 38 38 38 62 62 62 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 38, Max flush: 62 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 56 56 56 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 35 35 35 55 55 55 F F F 3 060-079 28 28 28 43 43 43 F F F 4 080-099 37 37 37 57 57 57 F F F 5 100-119 31 31 31 49 49 49 F F F 6 120-139 37 37 37 57 57 57 F F F 7 140-159 38 38 38 62 62 62 F F F 8 160-179 38 38 38 62 62 62 F F F 9 180-199 38 38 38 62 62 62 F F F 10 200-219 37 37 37 57 57 57 F F F 11 220-239 35 35 35 55 55 55 F F F 12 240-259 39 39 39 63 63 63 F F F 13 260-279 39 39 39 63 63 63 F F F 14 280-299 38 38 38 59 59 59 F F F 15 300-319 37 37 37 57 57 57 F F F 16 320-339 39 39 39 63 63 63 F F F 17 340-359 39 39 39 63 63 63 F F F 18 360-379 37 37 37 57 57 57 F F F 19 380-399 35 35 35 55 55 55 F F F 20 400-419 37 37 37 57 57 57 F F F 21 420-439 38 38 38 61 61 61 F F F 22 440-459 38 38 38 61 61 61 F F F 23 460-479 35 35 35 55 55 55 F F F 24 480-499 38 38 38 62 62 62 F F F 25 500-519 34 34 34 52 52 52 F F F 26 520-539 37 37 37 57 57 57 F F F 27 540-559 38 38 38 60 60 60 F F F 28 560-579 38 38 38 62 62 62 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 39, Max flush: 63 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.253490432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.297379672Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.298103018Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.29815561Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.298173483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.298191477Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348} {"msg":"Crucible 8dc426a5-7eca-4bb8-bebc-8d7ef4a6a396 has session id: 39a05783-0d64-492d-90e0-0a7afe7ba82c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.29876055Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Upstairs opts: Upstairs UUID: 8dc426a5-7eca-4bb8-bebc-8d7ef4a6a396, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.298802609Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Crucible stats registered with UUID: 8dc426a5-7eca-4bb8-bebc-8d7ef4a6a396","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.298825634Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:18.298842338Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299292565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299320461Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348} {"msg":"8dc426a5-7eca-4bb8-bebc-8d7ef4a6a396 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299366841Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299386205Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"0","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299419352Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"0","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299436386Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"1","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.29945341Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"1","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299469333Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"2","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299491358Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"2","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299519964Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"io task","client":"2","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299550531Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"io task","client":"0","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299582347Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62154)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299614674Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"io task","client":"0","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35002)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299657483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"io task","client":"2","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:18.299692891Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:18.299725028Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:18.299801294Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.299979873Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.30038405Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"0","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.300425249Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"2","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.301059076Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"0","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.3012178Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"2","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.301656835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:18.301805997Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.301649104Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49028)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.301715219Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.302816436Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"1","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.303393821Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"1","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"[0]R flush_numbers[0..12]: [56, 63, 55, 43, 57, 49, 57, 62, 62, 62, 57, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304030888Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"[0]R generation[0..12]: [36, 39, 35, 28, 37, 31, 37, 38, 38, 38, 37, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304065386Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.30408552Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"[1]R flush_numbers[0..12]: [56, 63, 55, 43, 57, 49, 57, 62, 62, 62, 57, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304103224Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"[1]R generation[0..12]: [36, 39, 35, 28, 37, 31, 37, 38, 38, 38, 37, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304119947Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304137341Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"[2]R flush_numbers[0..12]: [56, 63, 55, 43, 57, 49, 57, 62, 62, 62, 57, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304154275Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"[2]R generation[0..12]: [36, 39, 35, 28, 37, 31, 37, 38, 38, 38, 37, 35]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304171258Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304188362Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304203776Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304219639Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304235342Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304250846Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"":"downstairs","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.30427107Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","Wait for a query_work_queue command to finish before sending IO pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304294975Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"0","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304311219Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"1","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304327202Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"client":"2","":"downstairs","session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304343306Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304357919Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"8dc426a5-7eca-4bb8-bebc-8d7ef4a6a396 is now active with session: 39a05783-0d64-492d-90e0-0a7afe7ba82c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304373332Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304387655Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348,"session_id":"39a05783-0d64-492d-90e0-0a7afe7ba82c"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304402368Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348} {"msg":"Activated sub_volume 8dc426a5-7eca-4bb8-bebc-8d7ef4a6a396","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:28.304415751Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 379 len 12288 data: 4 4 4 02/30 Write block 250 len 36864 data: 4 4 4 3 3 2 2 3 3 03/30 Write block 36 len 16384 data: 4 6 7 6 04/30 Write block 305 len 20480 data: 3 2 2 2 2 05/30 Write block 229 len 32768 data: 6 7 4 3 4 5 5 4 06/30 Write block 202 len 4096 data: 3 07/30 Read block 511 len 4096 08/30 Write block 163 len 20480 data: 6 6 11 9 5 09/30 Read block 564 len 36864 10/30 Flush 11/30 Write block 444 len 40960 data: 8 8 8 5 3 4 4 4 4 4 12/30 Write block 490 len 8192 data: 8 8 13/30 Read block 107 len 20480 14/30 Flush 15/30 Write block 76 len 40960 data: 5 4 4 2 2 2 2 3 3 3 16/30 Write block 568 len 8192 data: 5 5 17/30 Flush 18/30 Write block 296 len 4096 data: 5 19/30 Write block 442 len 32768 data: 8 8 9 9 9 6 4 5 20/30 Write block 194 len 8192 data: 5 4 21/30 Read block 140 len 8192 22/30 Write block 42 len 12288 data: 4 4 5 23/30 Read block 349 len 16384 24/30 Flush 25/30 Write block 118 len 28672 data: 5 5 5 5 7 6 4 26/30 Read block 146 len 8192 27/30 Write block 423 len 4096 data: 4 28/30 Write block 105 len 16384 data: 3 3 3 2 29/30 Write block 67 len 4096 data: 3 30/30 Write block 467 len 36864 data: 6 6 5 5 4 6 6 6 6 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 1 1 5 Done 3 5 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:28.321968149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1348} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 56 56 56 F F F 1 020-039 40 40 40 64 64 64 F F F 2 040-059 40 40 40 67 67 67 F F F 3 060-079 40 40 40 66 66 66 T T T 4 080-099 40 40 40 66 66 66 F F F 5 100-119 31 31 31 49 49 49 T T T 6 120-139 37 37 37 57 57 57 T T T 7 140-159 38 38 38 62 62 62 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 40 40 40 67 67 67 F F F 10 200-219 40 40 40 64 64 64 F F F 11 220-239 40 40 40 64 64 64 F F F 12 240-259 40 40 40 64 64 64 F F F 13 260-279 39 39 39 63 63 63 F F F 14 280-299 40 40 40 67 67 67 F F F 15 300-319 40 40 40 64 64 64 F F F 16 320-339 39 39 39 63 63 63 F F F 17 340-359 39 39 39 63 63 63 F F F 18 360-379 40 40 40 64 64 64 F F F 19 380-399 40 40 40 64 64 64 F F F 20 400-419 37 37 37 57 57 57 F F F 21 420-439 38 38 38 61 61 61 T T T 22 440-459 40 40 40 67 67 67 F F F 23 460-479 35 35 35 55 55 55 T T T 24 480-499 40 40 40 65 65 65 F F F 25 500-519 34 34 34 52 52 52 F F F 26 520-539 37 37 37 57 57 57 F F F 27 540-559 38 38 38 60 60 60 F F F 28 560-579 40 40 40 66 66 66 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 40, Max flush: 67 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 56 56 56 F F F 1 020-039 40 40 40 64 64 64 F F F 2 040-059 40 40 40 67 67 67 F F F 3 060-079 41 41 41 68 68 68 F F F 4 080-099 40 40 40 66 66 66 F F F 5 100-119 41 41 41 68 68 68 F F F 6 120-139 41 41 41 68 68 68 F F F 7 140-159 38 38 38 62 62 62 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 40 40 40 67 67 67 F F F 10 200-219 40 40 40 64 64 64 F F F 11 220-239 40 40 40 64 64 64 F F F 12 240-259 40 40 40 64 64 64 F F F 13 260-279 39 39 39 63 63 63 F F F 14 280-299 40 40 40 67 67 67 F F F 15 300-319 40 40 40 64 64 64 F F F 16 320-339 39 39 39 63 63 63 F F F 17 340-359 39 39 39 63 63 63 F F F 18 360-379 40 40 40 64 64 64 F F F 19 380-399 40 40 40 64 64 64 F F F 20 400-419 37 37 37 57 57 57 F F F 21 420-439 41 41 41 68 68 68 F F F 22 440-459 40 40 40 67 67 67 F F F 23 460-479 41 41 41 68 68 68 F F F 24 480-499 40 40 40 65 65 65 F F F 25 500-519 34 34 34 52 52 52 F F F 26 520-539 37 37 37 57 57 57 F F F 27 540-559 38 38 38 60 60 60 F F F 28 560-579 40 40 40 66 66 66 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 41, Max flush: 68 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.561846667Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.605900873Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.606775001Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.606818471Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.606832864Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.606847047Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356} {"msg":"Crucible f7be327b-ad40-4737-bdac-6f9a1963cbf6 has session id: 3dfec4f1-c328-49d9-8161-75e4310ce339","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.607462069Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Upstairs opts: Upstairs UUID: f7be327b-ad40-4737-bdac-6f9a1963cbf6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.607519002Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Crucible stats registered with UUID: f7be327b-ad40-4737-bdac-6f9a1963cbf6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.607543327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:32.607563631Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608078052Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608104618Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356} {"msg":"f7be327b-ad40-4737-bdac-6f9a1963cbf6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608170582Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608189406Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"0","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608216962Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"0","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608230815Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"1","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608244558Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"1","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608262862Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"2","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608287868Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"2","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608312283Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"io task","client":"2","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608333858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"io task","client":"1","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608358253Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"io task","client":"0","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"ds_connection connected from Ok(127.0.0.1:35549)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608432689Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"io task","client":"2","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"ds_connection connected from Ok(127.0.0.1:46441)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608695825Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"io task","client":"1","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"ds_connection connected from Ok(127.0.0.1:48818)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.608735324Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"io task","client":"0","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.609279371Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"1","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.60931852Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"0","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.610028123Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"1","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.610056249Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"0","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.610333649Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"2","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.610606868Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.610718842Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.610983679Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"2","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"[0]R flush_numbers[0..12]: [56, 64, 67, 68, 66, 68, 68, 62, 64, 67, 64, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611513934Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"[0]R generation[0..12]: [36, 40, 40, 41, 40, 41, 41, 38, 40, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611538339Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611553332Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"[1]R flush_numbers[0..12]: [56, 64, 67, 68, 66, 68, 68, 62, 64, 67, 64, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611567245Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"[1]R generation[0..12]: [36, 40, 40, 41, 40, 41, 41, 38, 40, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611580698Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611594481Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"[2]R flush_numbers[0..12]: [56, 64, 67, 68, 66, 68, 68, 62, 64, 67, 64, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611608184Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"[2]R generation[0..12]: [36, 40, 40, 41, 40, 41, 41, 38, 40, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611621607Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611639841Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611653084Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611665866Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611678249Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611690902Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611710086Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611721589Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"0","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611734381Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"1","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611746924Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"client":"2","":"downstairs","session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611759507Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611771429Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"f7be327b-ad40-4737-bdac-6f9a1963cbf6 is now active with session: 3dfec4f1-c328-49d9-8161-75e4310ce339","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611783902Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611795524Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356,"session_id":"3dfec4f1-c328-49d9-8161-75e4310ce339"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611807347Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356} {"msg":"Activated sub_volume f7be327b-ad40-4737-bdac-6f9a1963cbf6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:32.611818439Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 64 len 24576 data: 4 3 3 4 3 3 02/30 Read block 81 len 20480 03/30 Read block 139 len 12288 04/30 Write block 16 len 28672 data: 2 2 2 3 5 4 5 05/30 Write block 208 len 20480 data: 5 4 3 3 3 06/30 Read block 430 len 16384 07/30 Write block 560 len 24576 data: 6 6 6 6 5 4 08/30 Read block 432 len 4096 09/30 Read block 91 len 32768 10/30 Write block 175 len 24576 data: 5 5 6 6 8 9 11/30 Read block 84 len 16384 12/30 Read block 16 len 32768 13/30 Read block 204 len 32768 14/30 Write block 276 len 8192 data: 4 3 15/30 Read block 307 len 16384 16/30 Write block 53 len 36864 data: 7 7 6 6 6 5 4 4 3 17/30 Read block 101 len 4096 18/30 Write block 554 len 36864 data: 3 3 3 3 5 5 7 7 7 19/30 Read block 348 len 8192 20/30 Write block 33 len 12288 data: 3 2 3 21/30 Read block 517 len 32768 22/30 Write block 179 len 32768 data: 9 10 7 7 7 8 8 7 23/30 Read block 6 len 24576 24/30 Write block 189 len 16384 data: 6 5 4 4 25/30 Read block 111 len 28672 26/30 Read block 127 len 8192 27/30 Write block 31 len 36864 data: 2 2 4 3 4 5 7 8 7 28/30 Read block 220 len 16384 29/30 Write block 246 len 32768 data: 3 3 2 3 5 5 5 4 30/30 Write block 68 len 36864 data: 4 4 4 3 6 6 7 6 6 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 8 Done Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 29 28 28 85 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:32.63351122Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1356} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 56 56 56 T T T 1 020-039 40 40 40 64 64 64 T T T 2 040-059 40 40 40 67 67 67 T T T 3 060-079 41 41 41 68 68 68 T T T 4 080-099 40 40 40 66 66 66 F F F 5 100-119 41 41 41 68 68 68 F F F 6 120-139 41 41 41 68 68 68 F F F 7 140-159 38 38 38 62 62 62 F F F 8 160-179 40 40 40 64 64 64 T T T 9 180-199 40 40 40 67 67 67 T T T 10 200-219 40 40 40 64 64 64 T T T 11 220-239 40 40 40 64 64 64 F F F 12 240-259 40 40 40 64 64 64 T T T 13 260-279 39 39 39 63 63 63 T T T 14 280-299 40 40 40 67 67 67 F F F 15 300-319 40 40 40 64 64 64 F F F 16 320-339 39 39 39 63 63 63 F F F 17 340-359 39 39 39 63 63 63 F F F 18 360-379 40 40 40 64 64 64 F F F 19 380-399 40 40 40 64 64 64 F F F 20 400-419 37 37 37 57 57 57 F F F 21 420-439 41 41 41 68 68 68 F F F 22 440-459 40 40 40 67 67 67 F F F 23 460-479 41 41 41 68 68 68 F F F 24 480-499 40 40 40 65 65 65 F F F 25 500-519 34 34 34 52 52 52 F F F 26 520-539 37 37 37 57 57 57 F F F 27 540-559 38 38 38 60 60 60 T T T 28 560-579 40 40 40 66 66 66 T T T 29 580-599 33 33 33 51 51 51 F F F Max gen: 41, Max flush: 68 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 69 69 69 F F F 1 020-039 42 42 42 69 69 69 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 42 42 42 69 69 69 F F F 4 080-099 40 40 40 66 66 66 F F F 5 100-119 41 41 41 68 68 68 F F F 6 120-139 41 41 41 68 68 68 F F F 7 140-159 38 38 38 62 62 62 F F F 8 160-179 42 42 42 69 69 69 F F F 9 180-199 42 42 42 69 69 69 F F F 10 200-219 42 42 42 69 69 69 F F F 11 220-239 40 40 40 64 64 64 F F F 12 240-259 42 42 42 69 69 69 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 40 40 40 67 67 67 F F F 15 300-319 40 40 40 64 64 64 F F F 16 320-339 39 39 39 63 63 63 F F F 17 340-359 39 39 39 63 63 63 F F F 18 360-379 40 40 40 64 64 64 F F F 19 380-399 40 40 40 64 64 64 F F F 20 400-419 37 37 37 57 57 57 F F F 21 420-439 41 41 41 68 68 68 F F F 22 440-459 40 40 40 67 67 67 F F F 23 460-479 41 41 41 68 68 68 F F F 24 480-499 40 40 40 65 65 65 F F F 25 500-519 34 34 34 52 52 52 F F F 26 520-539 37 37 37 57 57 57 F F F 27 540-559 42 42 42 69 69 69 F F F 28 560-579 42 42 42 69 69 69 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 42, Max flush: 69 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.905699241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:36.948851292Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.949233324Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.981067593Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.981804472Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.98184068Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.981854333Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.981869056Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364} {"msg":"Crucible 70d62310-202e-4beb-8ee1-f0c46ec81639 has session id: cfe31bb0-66b1-4d92-841c-fbc49867b8b8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.982387968Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Upstairs opts: Upstairs UUID: 70d62310-202e-4beb-8ee1-f0c46ec81639, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.982430557Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Crucible stats registered with UUID: 70d62310-202e-4beb-8ee1-f0c46ec81639","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.982457623Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:36.982475997Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.982889596Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.982921633Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364} {"msg":"70d62310-202e-4beb-8ee1-f0c46ec81639 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.982976355Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.983002591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"0","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.983031697Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"0","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.983049131Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"1","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.983063834Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"1","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.983076877Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"2","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.98309075Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"2","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.983104193Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.983127288Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.983151693Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:65019)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.983349216Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:56574)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.983392215Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:60235)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.98341908Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.984048596Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"2","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.984076533Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"1","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.984139686Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"0","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.984745247Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"1","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.984772613Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"0","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.984791747Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"2","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985365681Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985401818Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"[0]R flush_numbers[0..12]: [69, 69, 69, 69, 66, 68, 68, 62, 69, 69, 69, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985460261Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"[0]R generation[0..12]: [42, 42, 42, 42, 40, 41, 41, 38, 42, 42, 42, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985488197Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.98550391Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"[1]R flush_numbers[0..12]: [69, 69, 69, 69, 66, 68, 68, 62, 69, 69, 69, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985518053Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"[1]R generation[0..12]: [42, 42, 42, 42, 40, 41, 41, 38, 42, 42, 42, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985531296Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985544829Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"[2]R flush_numbers[0..12]: [69, 69, 69, 69, 66, 68, 68, 62, 69, 69, 69, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985557972Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"[2]R generation[0..12]: [42, 42, 42, 42, 40, 41, 41, 38, 42, 42, 42, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985571935Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985585508Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Max found gen is 43","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985597981Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Generation requested: 44 >= found:43","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985610753Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Next flush: 70","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985629858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985647001Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"":"downstairs","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985660934Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985672827Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"0","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.9856859Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"1","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985698883Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"client":"2","":"downstairs","session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985711685Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985730019Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"70d62310-202e-4beb-8ee1-f0c46ec81639 is now active with session: cfe31bb0-66b1-4d92-841c-fbc49867b8b8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985742812Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985754555Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364,"session_id":"cfe31bb0-66b1-4d92-841c-fbc49867b8b8"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.985766387Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364} {"msg":"Activated sub_volume 70d62310-202e-4beb-8ee1-f0c46ec81639","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:36.9857774Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 97 len 12288 data: 3 3 3 02/30 Read block 558 len 4096 03/30 Read block 275 len 20480 04/30 Read block 196 len 16384 05/30 Write block 513 len 40960 data: 5 6 7 7 8 8 6 6 5 6 06/30 Read block 519 len 8192 07/30 Write block 290 len 12288 data: 5 3 2 08/30 Read block 138 len 40960 09/30 Write block 297 len 24576 data: 5 6 5 6 6 4 10/30 Read block 377 len 8192 11/30 Read block 63 len 12288 12/30 Write block 467 len 32768 data: 7 7 6 6 5 7 7 7 13/30 Read block 279 len 24576 14/30 Read block 235 len 4096 15/30 Write block 296 len 24576 data: 6 6 7 6 7 7 16/30 Read block 569 len 32768 17/30 Write block 483 len 16384 data: 4 6 6 7 18/30 Write block 70 len 8192 data: 5 4 19/30 Read block 147 len 24576 20/30 Write block 197 len 36864 data: 3 3 3 4 4 4 5 5 5 21/30 Read block 521 len 20480 22/30 Read block 313 len 28672 23/30 Write block 138 len 8192 data: 4 5 24/30 Write block 416 len 16384 data: 3 3 3 3 25/30 Write block 542 len 16384 data: 3 3 3 3 26/30 Write block 151 len 24576 data: 5 4 4 6 7 8 27/30 Write block 10 len 12288 data: 5 4 5 28/30 Read block 316 len 40960 29/30 Read block 44 len 16384 30/30 Write block 375 len 20480 data: 5 4 3 4 5 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Write 10 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:37.005713121Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1364} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 69 69 69 T T T 1 020-039 42 42 42 69 69 69 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 42 42 42 69 69 69 T T T 4 080-099 40 40 40 66 66 66 T T T 5 100-119 41 41 41 68 68 68 F F F 6 120-139 41 41 41 68 68 68 T T T 7 140-159 38 38 38 62 62 62 T T T 8 160-179 42 42 42 69 69 69 F F F 9 180-199 42 42 42 69 69 69 T T T 10 200-219 42 42 42 69 69 69 T T T 11 220-239 40 40 40 64 64 64 F F F 12 240-259 42 42 42 69 69 69 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 40 40 40 67 67 67 T T T 15 300-319 40 40 40 64 64 64 T T T 16 320-339 39 39 39 63 63 63 F F F 17 340-359 39 39 39 63 63 63 F F F 18 360-379 40 40 40 64 64 64 T T T 19 380-399 40 40 40 64 64 64 F F F 20 400-419 37 37 37 57 57 57 T T T 21 420-439 41 41 41 68 68 68 F F F 22 440-459 40 40 40 67 67 67 F F F 23 460-479 41 41 41 68 68 68 T T T 24 480-499 40 40 40 65 65 65 T T T 25 500-519 34 34 34 52 52 52 T T T 26 520-539 37 37 37 57 57 57 T T T 27 540-559 42 42 42 69 69 69 T T T 28 560-579 42 42 42 69 69 69 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 42, Max flush: 69 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 70 70 70 F F F 1 020-039 42 42 42 69 69 69 F F F 2 040-059 42 42 42 69 69 69 F F F 3 060-079 43 43 43 70 70 70 F F F 4 080-099 43 43 43 70 70 70 F F F 5 100-119 41 41 41 68 68 68 F F F 6 120-139 43 43 43 70 70 70 F F F 7 140-159 43 43 43 70 70 70 F F F 8 160-179 42 42 42 69 69 69 F F F 9 180-199 43 43 43 70 70 70 F F F 10 200-219 43 43 43 70 70 70 F F F 11 220-239 40 40 40 64 64 64 F F F 12 240-259 42 42 42 69 69 69 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 43 43 43 70 70 70 F F F 15 300-319 43 43 43 70 70 70 F F F 16 320-339 39 39 39 63 63 63 F F F 17 340-359 39 39 39 63 63 63 F F F 18 360-379 43 43 43 70 70 70 F F F 19 380-399 40 40 40 64 64 64 F F F 20 400-419 43 43 43 70 70 70 F F F 21 420-439 41 41 41 68 68 68 F F F 22 440-459 40 40 40 67 67 67 F F F 23 460-479 43 43 43 70 70 70 F F F 24 480-499 43 43 43 70 70 70 F F F 25 500-519 43 43 43 70 70 70 F F F 26 520-539 43 43 43 70 70 70 F F F 27 540-559 43 43 43 70 70 70 F F F 28 560-579 42 42 42 69 69 69 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 43, Max flush: 70 Begin loop 23 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.330904858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:41.374214033Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.374611298Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.40645703Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.407114152Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.407155961Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.407177885Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.40719761Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372} {"msg":"Crucible fbfa0160-dc3d-492c-99cd-03469dab5d82 has session id: 41c1060a-599e-4e21-b9cb-94db1b218310","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.407853331Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Upstairs opts: Upstairs UUID: fbfa0160-dc3d-492c-99cd-03469dab5d82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.407901371Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Crucible stats registered with UUID: fbfa0160-dc3d-492c-99cd-03469dab5d82","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.407926637Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:41.407949101Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408384346Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.40840425Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372} {"msg":"fbfa0160-dc3d-492c-99cd-03469dab5d82 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408468204Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408484907Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"0","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408513253Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"0","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408527386Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"1","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408541039Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"1","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408553622Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"2","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408567275Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"2","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408640721Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"io task","client":"2","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408672008Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"io task","client":"0","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408697533Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"io task","client":"1","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"ds_connection connected from Ok(127.0.0.1:47323)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408723619Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"io task","client":"2","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"ds_connection connected from Ok(127.0.0.1:40646)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.408751565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"io task","client":"0","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"ds_connection connected from Ok(127.0.0.1:51652)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.40877668Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"io task","client":"1","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.409462768Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"1","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.409506037Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"2","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.409650068Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"0","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410185904Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"1","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410225223Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"2","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410252398Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"0","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410777172Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410810269Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"[0]R flush_numbers[0..12]: [70, 69, 69, 70, 70, 68, 70, 70, 69, 70, 70, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410836845Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"[0]R generation[0..12]: [43, 42, 42, 43, 43, 41, 43, 43, 42, 43, 43, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410853798Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410868351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"[1]R flush_numbers[0..12]: [70, 69, 69, 70, 70, 68, 70, 70, 69, 70, 70, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410881974Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"[1]R generation[0..12]: [43, 42, 42, 43, 43, 41, 43, 43, 42, 43, 43, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410895327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.41090988Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"[2]R flush_numbers[0..12]: [70, 69, 69, 70, 70, 68, 70, 70, 69, 70, 70, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410923073Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"[2]R generation[0..12]: [43, 42, 42, 43, 43, 41, 43, 43, 42, 43, 43, 40]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410936286Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410956361Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.410979826Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Generation requested: 46 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.411003561Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:21:41.411028096Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.41104517Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"":"downstairs","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.411058122Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.411069915Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"0","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.411090699Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"1","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.411114715Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"client":"2","":"downstairs","session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.411130618Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.411142831Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"fbfa0160-dc3d-492c-99cd-03469dab5d82 is now active with session: 41c1060a-599e-4e21-b9cb-94db1b218310","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.411155323Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.411168766Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372,"session_id":"41c1060a-599e-4e21-b9cb-94db1b218310"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.411189931Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372} {"msg":"Activated sub_volume fbfa0160-dc3d-492c-99cd-03469dab5d82","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:41.411206884Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 367 len 8192 data: 3 3 02/30 Write block 556 len 20480 data: 4 4 6 6 8 03/30 Write block 399 len 32768 data: 5 5 4 3 4 4 5 5 04/30 Flush 05/30 Write block 463 len 4096 data: 5 06/30 Flush 07/30 Write block 533 len 28672 data: 3 3 3 3 3 2 2 08/30 Read block 342 len 20480 09/30 Write block 39 len 24576 data: 8 5 5 5 5 6 10/30 Read block 234 len 20480 11/30 Write block 558 len 32768 data: 7 7 9 8 8 7 6 5 12/30 Read block 564 len 8192 13/30 Write block 184 len 4096 data: 9 14/30 Write block 70 len 24576 data: 6 5 7 7 8 7 15/30 Write block 290 len 36864 data: 6 4 3 3 4 4 7 7 8 16/30 Read block 165 len 8192 17/30 Flush 18/30 Write block 512 len 16384 data: 6 6 7 8 19/30 Write block 392 len 24576 data: 3 4 3 4 5 5 20/30 Read block 40 len 32768 21/30 Read block 289 len 28672 22/30 Write block 232 len 40960 data: 4 5 6 6 5 4 4 3 3 2 23/30 Write block 395 len 40960 data: 5 6 6 5 6 6 5 4 5 5 24/30 Read block 311 len 28672 25/30 Read block 167 len 16384 26/30 Read block 554 len 16384 27/30 Write block 450 len 24576 data: 5 5 5 5 4 3 28/30 Read block 354 len 28672 29/30 Read block 328 len 40960 30/30 Read block 577 len 4096 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 4 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 13 13 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:41.432282212Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1372} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 70 70 70 F F F 1 020-039 46 46 46 73 73 73 F F F 2 040-059 46 46 46 73 73 73 F F F 3 060-079 46 46 46 73 73 73 F F F 4 080-099 43 43 43 70 70 70 F F F 5 100-119 41 41 41 68 68 68 F F F 6 120-139 43 43 43 70 70 70 F F F 7 140-159 43 43 43 70 70 70 F F F 8 160-179 42 42 42 69 69 69 F F F 9 180-199 46 46 46 73 73 73 F F F 10 200-219 43 43 43 70 70 70 F F F 11 220-239 40 40 40 64 64 64 T T T 12 240-259 42 42 42 69 69 69 T T T 13 260-279 42 42 42 69 69 69 F F F 14 280-299 46 46 46 73 73 73 F F F 15 300-319 43 43 43 70 70 70 F F F 16 320-339 39 39 39 63 63 63 F F F 17 340-359 39 39 39 63 63 63 F F F 18 360-379 46 46 46 71 71 71 F F F 19 380-399 46 46 46 71 71 71 T T T 20 400-419 46 46 46 71 71 71 T T T 21 420-439 41 41 41 68 68 68 F F F 22 440-459 40 40 40 67 67 67 T T T 23 460-479 46 46 46 72 72 72 F F F 24 480-499 43 43 43 70 70 70 F F F 25 500-519 43 43 43 70 70 70 T T T 26 520-539 46 46 46 73 73 73 F F F 27 540-559 46 46 46 73 73 73 F F F 28 560-579 46 46 46 73 73 73 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 46, Max flush: 73 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 70 70 70 F F F 1 020-039 46 46 46 73 73 73 F F F 2 040-059 46 46 46 73 73 73 F F F 3 060-079 46 46 46 73 73 73 F F F 4 080-099 43 43 43 70 70 70 F F F 5 100-119 41 41 41 68 68 68 F F F 6 120-139 43 43 43 70 70 70 F F F 7 140-159 43 43 43 70 70 70 F F F 8 160-179 42 42 42 69 69 69 F F F 9 180-199 46 46 46 73 73 73 F F F 10 200-219 43 43 43 70 70 70 F F F 11 220-239 47 47 47 74 74 74 F F F 12 240-259 47 47 47 74 74 74 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 46 46 46 73 73 73 F F F 15 300-319 43 43 43 70 70 70 F F F 16 320-339 39 39 39 63 63 63 F F F 17 340-359 39 39 39 63 63 63 F F F 18 360-379 46 46 46 71 71 71 F F F 19 380-399 47 47 47 74 74 74 F F F 20 400-419 47 47 47 74 74 74 F F F 21 420-439 41 41 41 68 68 68 F F F 22 440-459 47 47 47 74 74 74 F F F 23 460-479 46 46 46 72 72 72 F F F 24 480-499 43 43 43 70 70 70 F F F 25 500-519 47 47 47 74 74 74 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 46 46 46 73 73 73 F F F 28 560-579 46 46 46 73 73 73 F F F 29 580-599 33 33 33 51 51 51 F F F Max gen: 47, Max flush: 74 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.704882721Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.748758898Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.749451008Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.749493327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.74950786Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.749522263Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380} {"msg":"Crucible ace2d0a2-1463-4d02-aaee-cde67f15b225 has session id: 5b01c6dc-6d0c-47cb-aaf9-bfadec185942","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.750128424Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Upstairs opts: Upstairs UUID: ace2d0a2-1463-4d02-aaee-cde67f15b225, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.750174624Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Crucible stats registered with UUID: ace2d0a2-1463-4d02-aaee-cde67f15b225","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.750198749Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:45.750215593Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.750679723Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.75071103Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380} {"msg":"ace2d0a2-1463-4d02-aaee-cde67f15b225 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.750777614Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.75080589Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"0","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.750842718Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"0","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.750870154Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"1","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.75089854Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"1","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.750925986Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"2","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.750954132Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"2","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.750978447Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"io task","client":"2","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.751040881Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"io task","client":"0","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.751071357Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"io task","client":"1","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"ds_connection connected from Ok(127.0.0.1:43306)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.751097373Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"io task","client":"2","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"ds_connection connected from Ok(127.0.0.1:62186)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.751124629Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"io task","client":"0","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"ds_connection connected from Ok(127.0.0.1:33723)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.751307828Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"io task","client":"1","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.751641911Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"2","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.751679088Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"0","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.752262184Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"2","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.752372518Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"0","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.752822265Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.75293639Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.753805637Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"1","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.754337702Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"1","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"[0]R flush_numbers[0..12]: [70, 73, 73, 73, 70, 68, 70, 70, 69, 73, 70, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.754901704Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"[0]R generation[0..12]: [43, 46, 46, 46, 43, 41, 43, 43, 42, 46, 43, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.754922008Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.754936591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"[1]R flush_numbers[0..12]: [70, 73, 73, 73, 70, 68, 70, 70, 69, 73, 70, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.754949994Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"[1]R generation[0..12]: [43, 46, 46, 46, 43, 41, 43, 43, 42, 46, 43, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.754963277Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.75497683Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"[2]R flush_numbers[0..12]: [70, 73, 73, 73, 70, 68, 70, 70, 69, 73, 70, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.754990133Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"[2]R generation[0..12]: [43, 46, 46, 46, 43, 41, 43, 43, 42, 46, 43, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755003396Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.75502124Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755034283Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755047205Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755059698Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755072221Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"":"downstairs","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755084873Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755096536Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":Wait for a query_work_queue command to finish before sending IO 1380,"client":"0","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755121921Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"1","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755137115Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"client":"2","":"downstairs","session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755150468Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.75516231Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"ace2d0a2-1463-4d02-aaee-cde67f15b225 is now active with session: 5b01c6dc-6d0c-47cb-aaf9-bfadec185942","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755174903Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755186225Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380,"session_id":"5b01c6dc-6d0c-47cb-aaf9-bfadec185942"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.755197888Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380} {"msg":"Activated sub_volume ace2d0a2-1463-4d02-aaee-cde67f15b225","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:21:45.75520883Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 177 len 32768 data: 7 7 10 11 8 8 8 10 02/30 Read block 392 len 24576 03/30 Write block 389 len 8192 data: 2 3 04/30 Read block 351 len 40960 05/30 Write block 86 len 28672 data: 4 4 4 3 5 4 4 06/30 Write block 200 len 8192 data: 5 5 07/30 Read block 48 len 36864 08/30 Read block 295 len 16384 09/30 Read block 473 len 32768 10/30 Write block 325 len 12288 data: 4 4 4 11/30 Read block 84 len 16384 12/30 Read block 474 len 36864 13/30 Read block 513 len 20480 14/30 Read block 251 len 24576 15/30 Read block 2 len 28672 16/30 Read block 463 len 16384 17/30 Flush 18/30 Write block 160 len 28672 data: 6 7 5 7 7 12 10 19/30 Read block 504 len 4096 20/30 Read block 12 len 4096 21/30 Write block 387 len 32768 data: 3 3 3 4 3 4 5 4 22/30 Read block 416 len 8192 23/30 Flush 24/30 Write block 306 len 36864 data: 3 3 3 3 2 4 4 4 4 25/30 Read block 535 len 28672 26/30 Read block 208 len 20480 27/30 Write block 407 len 8192 data: 8 6 28/30 Write block 468 len 8192 data: 8 7 29/30 Write block 581 len 20480 data: 5 4 4 6 6 30/30 Read block 478 len 4096 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 7 6 5 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:21:45.776266914Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1380} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 70 70 70 F F F 1 020-039 46 46 46 73 73 73 F F F 2 040-059 46 46 46 73 73 73 F F F 3 060-079 46 46 46 73 73 73 F F F 4 080-099 48 48 48 75 75 75 F F F 5 100-119 41 41 41 68 68 68 F F F 6 120-139 43 43 43 70 70 70 F F F 7 140-159 43 43 43 70 70 70 F F F 8 160-179 48 48 48 76 76 76 F F F 9 180-199 48 48 48 75 75 75 F F F 10 200-219 48 48 48 75 75 75 F F F 11 220-239 47 47 47 74 74 74 F F F 12 240-259 47 47 47 74 74 74 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 46 46 46 73 73 73 F F F 15 300-319 43 43 43 70 70 70 T T T 16 320-339 48 48 48 75 75 75 F F F 17 340-359 39 39 39 63 63 63 F F F 18 360-379 46 46 46 71 71 71 F F F 19 380-399 48 48 48 76 76 76 F F F 20 400-419 47 47 47 74 74 74 T T T 21 420-439 41 41 41 68 68 68 F F F 22 440-459 47 47 47 74 74 74 F F F 23 460-479 46 46 46 72 72 72 T T T 24 480-499 43 43 43 70 70 70 F F F 25 500-519 47 47 47 74 74 74 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 46 46 46 73 73 73 F F F 28 560-579 46 46 46 73 73 73 F F F 29 580-599 33 33 33 51 51 51 T T T Max gen: 48, Max flush: 76 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 70 70 70 F F F 1 020-039 46 46 46 73 73 73 F F F 2 040-059 46 46 46 73 73 73 F F F 3 060-079 46 46 46 73 73 73 F F F 4 080-099 48 48 48 75 75 75 F F F 5 100-119 41 41 41 68 68 68 F F F 6 120-139 43 43 43 70 70 70 F F F 7 140-159 43 43 43 70 70 70 F F F 8 160-179 48 48 48 76 76 76 F F F 9 180-199 48 48 48 75 75 75 F F F 10 200-219 48 48 48 75 75 75 F F F 11 220-239 47 47 47 74 74 74 F F F 12 240-259 47 47 47 74 74 74 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 46 46 46 73 73 73 F F F 15 300-319 49 49 49 77 77 77 F F F 16 320-339 48 48 48 75 75 75 F F F 17 340-359 39 39 39 63 63 63 F F F 18 360-379 46 46 46 71 71 71 F F F 19 380-399 48 48 48 76 76 76 F F F 20 400-419 49 49 49 77 77 77 F F F 21 420-439 41 41 41 68 68 68 F F F 22 440-459 47 47 47 74 74 74 F F F 23 460-479 49 49 49 77 77 77 F F F 24 480-499 43 43 43 70 70 70 F F F 25 500-519 47 47 47 74 74 74 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 46 46 46 73 73 73 F F F 28 560-579 46 46 46 73 73 73 F F F 29 580-599 49 49 49 77 77 77 F F F Max gen: 49, Max flush: 77 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.007037949Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.051335307Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.051986787Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.052031957Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.052048011Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.052062964Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388} {"msg":"Crucible 6b900e33-1fc2-4c41-863d-9356522b5bc0 has session id: a9d96e3e-b506-431e-8c80-c65572f2c3bc","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.052631947Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Upstairs opts: Upstairs UUID: 6b900e33-1fc2-4c41-863d-9356522b5bc0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.052676106Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Crucible stats registered with UUID: 6b900e33-1fc2-4c41-863d-9356522b5bc0","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.052700512Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:00.052715285Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.053267974Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.053292839Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388} {"msg":"6b900e33-1fc2-4c41-863d-9356522b5bc0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.053318265Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.053333458Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"0","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.053363804Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"0","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.053378428Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"1","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.053392931Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"1","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.053413075Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"2","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.05343438Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"2","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.053457195Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.0534816Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.053507075Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:52127)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.05357431Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:62976)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.053660039Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:52968)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.053734235Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.054447489Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"2","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.054474964Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"0","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.055144799Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"0","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.055172915Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"2","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.055696918Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.055791358Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"1","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.055899362Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.056327704Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"1","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"[0]R flush_numbers[0..12]: [70, 73, 73, 73, 75, 68, 70, 70, 76, 75, 75, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.056833743Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"[0]R generation[0..12]: [43, 46, 46, 46, 48, 41, 43, 43, 48, 48, 48, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.056852968Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.056867641Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"[1]R flush_numbers[0..12]: [70, 73, 73, 73, 75, 68, 70, 70, 76, 75, 75, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.056881174Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"[1]R generation[0..12]: [43, 46, 46, 46, 48, 41, 43, 43, 48, 48, 48, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.056907339Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.056933935Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"[2]R flush_numbers[0..12]: [70, 73, 73, 73, 75, 68, 70, 70, 76, 75, 75, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.056960111Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"[2]R generation[0..12]: [43, 46, 46, 46, 48, 41, 43, 43, 48, 48, 48, 47]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:22:00.056985766Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057033376Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057058562Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057082887Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057107703Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057130397Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"":"downstairs","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.05714442Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057159884Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"0","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057183489Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"1","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057208874Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"client":"2","":"downstairs","session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057231859Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057244672Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"6b900e33-1fc2-4c41-863d-9356522b5bc0 is now active with session: a9d96e3e-b506-431e-8c80-c65572f2c3bc","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057264806Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-17T22:22:00.057286221Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388,"session_id":"a9d96e3e-b506-431e-8c80-c65572f2c3bc"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057306825Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388} {"msg":"Activated sub_volume 6b900e33-1fc2-4c41-863d-9356522b5bc0","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:00.057325079Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 26 len 16384 data: 3 3 3 3 02/30 Write block 107 len 40960 data: 4 3 2 2 2 2 3 3 3 5 03/30 Read block 360 len 32768 04/30 Read block 63 len 20480 05/30 Read block 561 len 32768 06/30 Write block 183 len 32768 data: 9 11 9 8 7 7 7 6 07/30 Read block 47 len 16384 08/30 Read block 485 len 36864 09/30 Write block 200 len 8192 data: 6 6 10/30 Write block 39 len 28672 data: 9 6 6 6 6 7 4 11/30 Write block 349 len 32768 data: 3 3 3 2 3 3 4 4 12/30 Read block 94 len 32768 13/30 Flush 14/30 Write block 523 len 28672 data: 7 6 6 8 9 7 6 15/30 Write block 478 len 32768 data: 3 3 3 3 4 5 7 7 16/30 Flush 17/30 Write block 101 len 16384 data: 3 3 3 4 18/30 Write block 74 len 20480 data: 9 8 7 5 5 19/30 Write block 466 len 40960 data: 3 8 9 8 7 6 8 8 8 7 20/30 Read block 274 len 36864 21/30 Write block 83 len 36864 data: 4 4 4 5 5 5 4 6 5 22/30 Flush 23/30 Flush 24/30 Write block 176 len 28672 data: 6 8 8 11 12 9 9 25/30 Read block 41 len 12288 26/30 Read block 530 len 12288 27/30 Write block 113 len 24576 data: 4 4 4 6 5 6 28/30 Read block 281 len 24576 29/30 Write block 66 len 40960 data: 4 5 5 5 7 6 8 8 10 9 30/30 Write block 596 len 12288 data: 2 2 2 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:00.078537376Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1388} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 70 70 70 F F F 1 020-039 50 50 50 78 78 78 F F F 2 040-059 50 50 50 78 78 78 F F F 3 060-079 50 50 50 80 80 80 T T T 4 080-099 50 50 50 80 80 80 F F F 5 100-119 50 50 50 80 80 80 T T T 6 120-139 43 43 43 70 70 70 F F F 7 140-159 43 43 43 70 70 70 F F F 8 160-179 48 48 48 76 76 76 T T T 9 180-199 50 50 50 78 78 78 T T T 10 200-219 50 50 50 78 78 78 F F F 11 220-239 47 47 47 74 74 74 F F F 12 240-259 47 47 47 74 74 74 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 46 46 46 73 73 73 F F F 15 300-319 49 49 49 77 77 77 F F F 16 320-339 48 48 48 75 75 75 F F F 17 340-359 50 50 50 78 78 78 F F F 18 360-379 46 46 46 71 71 71 F F F 19 380-399 48 48 48 76 76 76 F F F 20 400-419 49 49 49 77 77 77 F F F 21 420-439 41 41 41 68 68 68 F F F 22 440-459 47 47 47 74 74 74 F F F 23 460-479 50 50 50 80 80 80 F F F 24 480-499 50 50 50 79 79 79 F F F 25 500-519 47 47 47 74 74 74 F F F 26 520-539 50 50 50 79 79 79 F F F 27 540-559 46 46 46 73 73 73 F F F 28 560-579 46 46 46 73 73 73 F F F 29 580-599 49 49 49 77 77 77 T T T Max gen: 50, Max flush: 80 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 70 70 70 F F F 1 020-039 50 50 50 78 78 78 F F F 2 040-059 50 50 50 78 78 78 F F F 3 060-079 51 51 51 81 81 81 F F F 4 080-099 50 50 50 80 80 80 F F F 5 100-119 51 51 51 81 81 81 F F F 6 120-139 43 43 43 70 70 70 F F F 7 140-159 43 43 43 70 70 70 F F F 8 160-179 51 51 51 81 81 81 F F F 9 180-199 51 51 51 81 81 81 F F F 10 200-219 50 50 50 78 78 78 F F F 11 220-239 47 47 47 74 74 74 F F F 12 240-259 47 47 47 74 74 74 F F F 13 260-279 42 42 42 69 69 69 F F F 14 280-299 46 46 46 73 73 73 F F F 15 300-319 49 49 49 77 77 77 F F F 16 320-339 48 48 48 75 75 75 F F F 17 340-359 50 50 50 78 78 78 F F F 18 360-379 46 46 46 71 71 71 F F F 19 380-399 48 48 48 76 76 76 F F F 20 400-419 49 49 49 77 77 77 F F F 21 420-439 41 41 41 68 68 68 F F F 22 440-459 47 47 47 74 74 74 F F F 23 460-479 50 50 50 80 80 80 F F F 24 480-499 50 50 50 79 79 79 F F F 25 500-519 47 47 47 74 74 74 F F F 26 520-539 50 50 50 79 79 79 F F F 27 540-559 46 46 46 73 73 73 F F F 28 560-579 46 46 46 73 73 73 F F F 29 580-599 51 51 51 81 81 81 F F F Max gen: 51, Max flush: 81 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.320685845Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:04.364100852Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.364581026Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.396260282Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.397029187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.397073047Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.39709017Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.397108534Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396} {"msg":"Crucible 6fe6ef62-740d-469b-8add-88708532c186 has session id: 2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.39773796Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Upstairs opts: Upstairs UUID: 6fe6ef62-740d-469b-8add-88708532c186, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.397778599Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Crucible stats registered with UUID: 6fe6ef62-740d-469b-8add-88708532c186","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.397796903Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:04.397810276Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398216304Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398240339Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396} {"msg":"6fe6ef62-740d-469b-8add-88708532c186 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398303142Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398321076Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"0","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398348662Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"0","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398362865Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"1","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.3983833Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"1","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398409075Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"2","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398437381Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"2","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398463077Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"io task","client":"2","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398491733Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"io task","client":"0","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398519039Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"io task","client":"1","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"ds_connection connected from Ok(127.0.0.1:54447)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398544544Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"io task","client":"2","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"ds_connection connected from Ok(127.0.0.1:47507)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398578802Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"io task","client":"1","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"ds_connection connected from Ok(127.0.0.1:39794)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.398610109Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"io task","client":"0","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.399444498Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"1","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.399484947Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"2","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.399518014Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"0","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400026994Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"1","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.40005486Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"0","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400073854Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"2","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400650649Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400685926Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"[0]R flush_numbers[0..12]: [70, 78, 78, 81, 80, 81, 70, 70, 81, 81, 78, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400744189Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"[0]R generation[0..12]: [43, 50, 50, 51, 50, 51, 43, 43, 51, 51, 50, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400772995Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400789229Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"[1]R flush_numbers[0..12]: [70, 78, 78, 81, 80, 81, 70, 70, 81, 81, 78, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400803592Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"[1]R generation[0..12]: [43, 50, 50, 51, 50, 51, 43, 43, 51, 51, 50, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400817335Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400830948Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"[2]R flush_numbers[0..12]: [70, 78, 78, 81, 80, 81, 70, 70, 81, 81, 78, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.40084434Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"[2]R generation[0..12]: [43, 50, 50, 51, 50, 51, 43, 43, 51, 51, 50, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400857523Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400870986Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400883829Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400897032Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400913246Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400926368Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400947643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400959235Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"0","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400972058Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"1","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400984781Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"client":"2","":"downstairs","session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.400997424Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.401009396Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"6fe6ef62-740d-469b-8add-88708532c186 is now active with session: 2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.401021879Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.401033261Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396,"session_id":"2497cd3b-8ff3-4e74-8fae-2baf7a4a3e54"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.401045104Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396} {"msg":"Activated sub_volume 6fe6ef62-740d-469b-8add-88708532c186","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:04.401055926Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 352 len 12288 data: 3 4 4 02/30 Read block 421 len 20480 03/30 Read block 166 len 36864 04/30 Read block 290 len 24576 05/30 Write block 552 len 28672 data: 3 3 4 4 5 5 8 06/30 Write block 508 len 16384 data: 5 6 6 4 07/30 Write block 462 len 28672 data: 5 6 5 5 4 9 10 08/30 Write block 452 len 16384 data: 6 6 5 4 09/30 Read block 450 len 8192 10/30 Write block 49 len 20480 data: 7 7 6 8 8 11/30 Write block 589 len 16384 data: 5 5 5 4 12/30 Read block 221 len 20480 13/30 Read block 30 len 28672 14/30 Write block 140 len 8192 data: 3 3 15/30 Write block 358 len 40960 data: 5 5 7 6 7 7 5 5 3 4 16/30 Write block 489 len 16384 data: 9 9 9 9 17/30 Write block 164 len 8192 data: 8 13 18/30 Write block 4 len 4096 data: 5 19/30 Write block 133 len 16384 data: 4 3 4 4 20/30 Read block 399 len 24576 21/30 Read block 22 len 40960 22/30 Write block 249 len 28672 data: 4 6 6 6 5 4 3 23/30 Write block 270 len 36864 data: 6 5 6 5 4 5 5 4 4 24/30 Write block 164 len 40960 data: 9 14 11 6 5 5 5 3 4 6 25/30 Write block 383 len 24576 data: 4 3 3 3 4 4 26/30 Read block 39 len 28672 27/30 Read block 226 len 40960 28/30 Read block 217 len 12288 29/30 Write block 313 len 40960 data: 5 5 5 6 6 6 5 5 3 3 30/30 Write block 29 len 16384 data: 4 3 3 3 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:04.419653389Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1396} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 70 70 70 T T T 1 020-039 50 50 50 78 78 78 T T T 2 040-059 50 50 50 78 78 78 T T T 3 060-079 51 51 51 81 81 81 F F F 4 080-099 50 50 50 80 80 80 F F F 5 100-119 51 51 51 81 81 81 F F F 6 120-139 43 43 43 70 70 70 T T T 7 140-159 43 43 43 70 70 70 T T T 8 160-179 51 51 51 81 81 81 T T T 9 180-199 51 51 51 81 81 81 F F F 10 200-219 50 50 50 78 78 78 F F F 11 220-239 47 47 47 74 74 74 F F F 12 240-259 47 47 47 74 74 74 T T T 13 260-279 42 42 42 69 69 69 T T T 14 280-299 46 46 46 73 73 73 F F F 15 300-319 49 49 49 77 77 77 T T T 16 320-339 48 48 48 75 75 75 T T T 17 340-359 50 50 50 78 78 78 T T T 18 360-379 46 46 46 71 71 71 T T T 19 380-399 48 48 48 76 76 76 T T T 20 400-419 49 49 49 77 77 77 F F F 21 420-439 41 41 41 68 68 68 F F F 22 440-459 47 47 47 74 74 74 T T T 23 460-479 50 50 50 80 80 80 T T T 24 480-499 50 50 50 79 79 79 T T T 25 500-519 47 47 47 74 74 74 T T T 26 520-539 50 50 50 79 79 79 F F F 27 540-559 46 46 46 73 73 73 T T T 28 560-579 46 46 46 73 73 73 F F F 29 580-599 51 51 51 81 81 81 T T T Max gen: 51, Max flush: 81 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 82 82 82 F F F 1 020-039 52 52 52 82 82 82 F F F 2 040-059 52 52 52 82 82 82 F F F 3 060-079 51 51 51 81 81 81 F F F 4 080-099 50 50 50 80 80 80 F F F 5 100-119 51 51 51 81 81 81 F F F 6 120-139 52 52 52 82 82 82 F F F 7 140-159 52 52 52 82 82 82 F F F 8 160-179 52 52 52 82 82 82 F F F 9 180-199 51 51 51 81 81 81 F F F 10 200-219 50 50 50 78 78 78 F F F 11 220-239 47 47 47 74 74 74 F F F 12 240-259 52 52 52 82 82 82 F F F 13 260-279 52 52 52 82 82 82 F F F 14 280-299 46 46 46 73 73 73 F F F 15 300-319 52 52 52 82 82 82 F F F 16 320-339 52 52 52 82 82 82 F F F 17 340-359 52 52 52 82 82 82 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 52 52 52 82 82 82 F F F 20 400-419 49 49 49 77 77 77 F F F 21 420-439 41 41 41 68 68 68 F F F 22 440-459 52 52 52 82 82 82 F F F 23 460-479 52 52 52 82 82 82 F F F 24 480-499 52 52 52 82 82 82 F F F 25 500-519 52 52 52 82 82 82 F F F 26 520-539 50 50 50 79 79 79 F F F 27 540-559 52 52 52 82 82 82 F F F 28 560-579 46 46 46 73 73 73 F F F 29 580-599 52 52 52 82 82 82 F F F Max gen: 52, Max flush: 82 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.770718779Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:08.813767116Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.814282468Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.845661909Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.846385045Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.846428084Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.846451769Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.846470203Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404} {"msg":"Crucible 55b5f1ea-4cbd-4c90-a88d-f933809417af has session id: a05f0483-c899-4d42-8ffd-2268fab4f0f7","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.846976712Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Upstairs opts: Upstairs UUID: 55b5f1ea-4cbd-4c90-a88d-f933809417af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847053849Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Crucible stats registered with UUID: 55b5f1ea-4cbd-4c90-a88d-f933809417af","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847084956Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:08.847100199Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847445994Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847469199Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404} {"msg":"55b5f1ea-4cbd-4c90-a88d-f933809417af active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847513788Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847539194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"0","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.84756818Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"0","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847582803Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"1","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847602227Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"1","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847626142Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"2","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847649927Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"2","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847678664Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"io task","client":"2","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847705759Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"io task","client":"0","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847731095Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"io task","client":"1","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33616)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847814023Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"io task","client":"2","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:37599)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847858202Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"io task","client":"1","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33537)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.847890369Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"io task","client":"0","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.848631709Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"2","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.848661236Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"0","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.848676889Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"1","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.849272758Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"1","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.849411587Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"0","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.849483093Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"2","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.849905324Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850040823Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"[0]R flush_numbers[0..12]: [82, 82, 82, 81, 80, 81, 82, 82, 82, 81, 78, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.85007069Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"[0]R generation[0..12]: [52, 52, 52, 51, 50, 51, 52, 52, 52, 51, 50, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850086593Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850100986Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"[1]R flush_numbers[0..12]: [82, 82, 82, 81, 80, 81, 82, 82, 82, 81, 78, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850114579Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"[1]R generation[0..12]: [52, 52, 52, 51, 50, 51, 52, 52, 52, 51, 50, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850127682Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850141305Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"[2]R flush_numbers[0..12]: [82, 82, 82, 81, 80, 81, 82, 82, 82, 81, 78, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850154508Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"[2]R generation[0..12]: [52, 52, 52, 51, 50, 51, 52, 52, 52, 51, 50, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850167581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850180933Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Max found gen is 53","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850193546Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Generation requested: 54 >= found:53","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850206199Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850218582Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850231004Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"":"downstairs","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850243637Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Transition from WaitQuorum to Active"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850266612Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"0","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850279775Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"1","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850292348Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"client":"2","":"downstairs","session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.85030492Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850316693Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"55b5f1ea-4cbd-4c90-a88d-f933809417af is now active with session: a05f0483-c899-4d42-8ffd-2268fab4f0f7","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850329296Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850340698Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404,"session_id":"a05f0483-c899-4d42-8ffd-2268fab4f0f7"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850352591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404} {"msg":"Activated sub_volume 55b5f1ea-4cbd-4c90-a88d-f933809417af","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:08.850363653Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 239 len 16384 data: 4 4 3 3 02/30 Read block 550 len 12288 03/30 Read block 428 len 16384 04/30 Write block 321 len 28672 data: 4 4 4 4 5 5 5 05/30 Write block 280 len 24576 data: 3 4 4 5 6 6 06/30 Read block 348 len 36864 07/30 Read block 446 len 4096 08/30 Read block 460 len 32768 09/30 Flush 10/30 Write block 21 len 12288 data: 5 6 4 11/30 Write block 265 len 12288 data: 12 10 9 12/30 Flush 13/30 Write block 586 len 4096 data: 4 14/30 Write block 253 len 32768 data: 6 5 4 3 4 4 6 6 15/30 Write block 148 len 8192 data: 4 4 16/30 Write block 11 len 4096 data: 5 17/30 Write block 485 len 12288 data: 8 8 8 18/30 Write block 469 len 12288 data: 9 8 7 19/30 Write block 178 len 16384 data: 9 12 13 10 20/30 Read block 211 len 16384 21/30 Write block 97 len 12288 data: 4 4 4 22/30 Write block 72 len 40960 data: 9 9 11 10 8 6 6 3 3 3 23/30 Write block 134 len 4096 data: 4 24/30 Read block 436 len 24576 25/30 Write block 251 len 4096 data: 7 26/30 Read block 587 len 16384 27/30 Read block 503 len 8192 28/30 Read block 444 len 28672 29/30 Read block 80 len 20480 30/30 Write block 453 len 8192 data: 7 6 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 1 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:08.867551542Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1404} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 82 82 82 T T T 1 020-039 54 54 54 84 84 84 F F F 2 040-059 52 52 52 82 82 82 F F F 3 060-079 51 51 51 81 81 81 T T T 4 080-099 50 50 50 80 80 80 T T T 5 100-119 51 51 51 81 81 81 F F F 6 120-139 52 52 52 82 82 82 T T T 7 140-159 52 52 52 82 82 82 T T T 8 160-179 52 52 52 82 82 82 T T T 9 180-199 51 51 51 81 81 81 T T T 10 200-219 50 50 50 78 78 78 F F F 11 220-239 54 54 54 83 83 83 F F F 12 240-259 54 54 54 83 83 83 T T T 13 260-279 54 54 54 84 84 84 T T T 14 280-299 54 54 54 83 83 83 F F F 15 300-319 52 52 52 82 82 82 F F F 16 320-339 54 54 54 83 83 83 F F F 17 340-359 52 52 52 82 82 82 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 52 52 52 82 82 82 F F F 20 400-419 49 49 49 77 77 77 F F F 21 420-439 41 41 41 68 68 68 F F F 22 440-459 52 52 52 82 82 82 T T T 23 460-479 52 52 52 82 82 82 T T T 24 480-499 52 52 52 82 82 82 T T T 25 500-519 52 52 52 82 82 82 F F F 26 520-539 50 50 50 79 79 79 F F F 27 540-559 52 52 52 82 82 82 F F F 28 560-579 46 46 46 73 73 73 F F F 29 580-599 52 52 52 82 82 82 T T T Max gen: 54, Max flush: 84 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 85 85 85 F F F 1 020-039 54 54 54 84 84 84 F F F 2 040-059 52 52 52 82 82 82 F F F 3 060-079 55 55 55 85 85 85 F F F 4 080-099 55 55 55 85 85 85 F F F 5 100-119 51 51 51 81 81 81 F F F 6 120-139 55 55 55 85 85 85 F F F 7 140-159 55 55 55 85 85 85 F F F 8 160-179 55 55 55 85 85 85 F F F 9 180-199 55 55 55 85 85 85 F F F 10 200-219 50 50 50 78 78 78 F F F 11 220-239 54 54 54 83 83 83 F F F 12 240-259 55 55 55 85 85 85 F F F 13 260-279 55 55 55 85 85 85 F F F 14 280-299 54 54 54 83 83 83 F F F 15 300-319 52 52 52 82 82 82 F F F 16 320-339 54 54 54 83 83 83 F F F 17 340-359 52 52 52 82 82 82 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 52 52 52 82 82 82 F F F 20 400-419 49 49 49 77 77 77 F F F 21 420-439 41 41 41 68 68 68 F F F 22 440-459 55 55 55 85 85 85 F F F 23 460-479 55 55 55 85 85 85 F F F 24 480-499 55 55 55 85 85 85 F F F 25 500-519 52 52 52 82 82 82 F F F 26 520-539 50 50 50 79 79 79 F F F 27 540-559 52 52 52 82 82 82 F F F 28 560-579 46 46 46 73 73 73 F F F 29 580-599 55 55 55 85 85 85 F F F Max gen: 55, Max flush: 85 Begin loop 28 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.178309235Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.224516265Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.225200563Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.225243252Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.225258155Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.225272528Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412} {"msg":"Crucible b8c9e096-48aa-49bd-b00b-0279c240dbdb has session id: 8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.22583722Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"Upstairs opts: Upstairs UUID: b8c9e096-48aa-49bd-b00b-0279c240dbdb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.22587997Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"Crucible stats registered with UUID: b8c9e096-48aa-49bd-b00b-0279c240dbdb","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.225905045Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:13.22592695Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226416865Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226446192Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412} {"msg":"b8c9e096-48aa-49bd-b00b-0279c240dbdb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226507725Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226526509Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"0","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226555245Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"0","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226569638Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"1","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226584111Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"1","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226596864Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"2","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226610487Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"2","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226630992Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"io task","client":"2","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226657337Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"io task","client":"0","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226674541Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"io task","client":"1","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46038)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226804599Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"io task","client":"2","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37558)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.226844858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"io task","client":"1","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35460)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.227008323Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"io task","client":"0","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.227555081Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"2","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.22759561Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"0","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.228220015Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"0","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.228371337Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"2","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.228517139Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"1","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.22880217Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.228911004Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229083211Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"1","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"[0]R flush_numbers[0..12]: [85, 84, 82, 85, 85, 81, 85, 85, 85, 85, 78, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229621957Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"[0]R generation[0..12]: [55, 54, 52, 55, 55, 51, 55, 55, 55, 55, 50, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229648032Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229666527Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"[1]R flush_numbers[0..12]: [85, 84, 82, 85, 85, 81, 85, 85, 85, 85, 78, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.22968332Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"[1]R generation[0..12]: [55, 54, 52, 55, 55, 51, 55, 55, 55, 55, 50, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229699924Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229716967Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"[2]R flush_numbers[0..12]: [85, 84, 82, 85, 85, 81, 85, 85, 85, 85, 78, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229733421Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"[2]R generation[0..12]: [55, 54, 52, 55, 55, 51, 55, 55, 55, 55, 50, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229754986Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229772759Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"Wait for a query_work_queue command to finish before sending IO msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229794864Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229811018Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229826541Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229842154Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"":"downstairs","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229857858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229872201Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"0","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229888234Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"1","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229904028Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"client":"2","":"downstairs","session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229919751Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229934394Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"b8c9e096-48aa-49bd-b00b-0279c240dbdb is now active with session: 8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229950038Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229964741Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412,"session_id":"8d4ae6dc-75a8-4453-aa8c-a5fff5ee8a3b"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:13.229979774Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412} {"msg":"Activated sub_volume b8c9e096-48aa-49bd-b00b-0279c240dbdb","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-17T22:22:13.23000664Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 205 len 4096 data: 6 03/30 Read block 247 len 32768 04/30 Read block 468 len 16384 05/30 Read block 378 len 8192 06/30 Read block 74 len 36864 07/30 Read block 361 len 16384 08/30 Read block 150 len 16384 09/30 Flush 10/30 Write block 402 len 28672 data: 5 6 6 6 6 9 7 11/30 Write block 285 len 16384 data: 7 6 4 4 12/30 Read block 54 len 8192 13/30 Read block 556 len 12288 14/30 Read block 28 len 12288 15/30 Write block 334 len 36864 data: 8 9 7 7 7 7 6 4 4 16/30 Read block 481 len 36864 17/30 Read block 34 len 32768 18/30 Read block 346 len 8192 19/30 Write block 437 len 16384 data: 5 4 6 7 20/30 Read block 403 len 12288 21/30 Read block 3 len 36864 22/30 Flush 23/30 Write block 429 len 20480 data: 5 6 5 5 6 24/30 Read block 37 len 16384 25/30 Write block 539 len 8192 data: 3 3 26/30 Flush 27/30 Write block 181 len 20480 data: 11 10 10 12 10 28/30 Write block 574 len 8192 data: 7 8 29/30 Read block 80 len 36864 30/30 Read block 471 len 40960 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:13.250191185Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1412} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 85 85 85 F F F 1 020-039 54 54 54 84 84 84 F F F 2 040-059 52 52 52 82 82 82 F F F 3 060-079 55 55 55 85 85 85 F F F 4 080-099 55 55 55 85 85 85 F F F 5 100-119 51 51 51 81 81 81 F F F 6 120-139 55 55 55 85 85 85 F F F 7 140-159 55 55 55 85 85 85 F F F 8 160-179 55 55 55 85 85 85 F F F 9 180-199 55 55 55 85 85 85 T T T 10 200-219 56 56 56 87 87 87 F F F 11 220-239 54 54 54 83 83 83 F F F 12 240-259 55 55 55 85 85 85 F F F 13 260-279 55 55 55 85 85 85 F F F 14 280-299 56 56 56 88 88 88 F F F 15 300-319 52 52 52 82 82 82 F F F 16 320-339 56 56 56 88 88 88 F F F 17 340-359 56 56 56 88 88 88 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 52 52 52 82 82 82 F F F 20 400-419 56 56 56 88 88 88 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 56 56 56 88 88 88 F F F 23 460-479 55 55 55 85 85 85 F F F 24 480-499 55 55 55 85 85 85 F F F 25 500-519 52 52 52 82 82 82 F F F 26 520-539 56 56 56 89 89 89 F F F 27 540-559 56 56 56 89 89 89 F F F 28 560-579 46 46 46 73 73 73 T T T 29 580-599 55 55 55 85 85 85 F F F Max gen: 56, Max flush: 89 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 85 85 85 F F F 1 020-039 54 54 54 84 84 84 F F F 2 040-059 52 52 52 82 82 82 F F F 3 060-079 55 55 55 85 85 85 F F F 4 080-099 55 55 55 85 85 85 F F F 5 100-119 51 51 51 81 81 81 F F F 6 120-139 55 55 55 85 85 85 F F F 7 140-159 55 55 55 85 85 85 F F F 8 160-179 55 55 55 85 85 85 F F F 9 180-199 57 57 57 90 90 90 F F F 10 200-219 56 56 56 87 87 87 F F F 11 220-239 54 54 54 83 83 83 F F F 12 240-259 55 55 55 85 85 85 F F F 13 260-279 55 55 55 85 85 85 F F F 14 280-299 56 56 56 88 88 88 F F F 15 300-319 52 52 52 82 82 82 F F F 16 320-339 56 56 56 88 88 88 F F F 17 340-359 56 56 56 88 88 88 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 52 52 52 82 82 82 F F F 20 400-419 56 56 56 88 88 88 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 56 56 56 88 88 88 F F F 23 460-479 55 55 55 85 85 85 F F F 24 480-499 55 55 55 85 85 85 F F F 25 500-519 52 52 52 82 82 82 F F F 26 520-539 56 56 56 89 89 89 F F F 27 540-559 56 56 56 89 89 89 F F F 28 560-579 57 57 57 90 90 90 F F F 29 580-599 55 55 55 85 85 85 F F F Max gen: 57, Max flush: 90 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.471764026Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.517975037Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.518722448Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.518763297Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.51877884Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.518793363Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420} {"msg":"Crucible 94fdbaf6-2ea7-476a-9a73-c954b4bdcf9a has session id: 100a22e7-adee-45b3-a6e8-daa0a06cce60","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.519432952Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Upstairs opts: Upstairs UUID: 94fdbaf6-2ea7-476a-9a73-c954b4bdcf9a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.519560129Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Crucible stats registered with UUID: 94fdbaf6-2ea7-476a-9a73-c954b4bdcf9a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.519597237Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:17.519615361Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.520070699Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.520094074Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420} {"msg":"94fdbaf6-2ea7-476a-9a73-c954b4bdcf9a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.52017002Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.520187824Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"0","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.52021621Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"0","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.520235785Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"1","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.52026183Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"1","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.520288566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"2","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.520315822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"2","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.520340107Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.520366873Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.520393609Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"ds_connection connected from Ok(127.0.0.1:51328)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.520485659Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"ds_connection connected from Ok(127.0.0.1:39303)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.520533239Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"ds_connection connected from Ok(127.0.0.1:49657)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.520686422Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.52137087Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"0","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.521409498Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"2","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.522092915Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"0","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.522130174Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"2","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.522328686Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"1","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.522848758Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.522877415Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.52290159Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"1","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"[0]R flush_numbers[0..12]: [85, 84, 82, 85, 85, 81, 85, 85, 85, 90, 87, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523416471Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"[0]R generation[0..12]: [55, 54, 52, 55, 55, 51, 55, 55, 55, 57, 56, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523442277Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.52345791Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"[1]R flush_numbers[0..12]: [85, 84, 82, 85, 85, 81, 85, 85, 85, 90, 87, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523472673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"[1]R generation[0..12]: [55, 54, 52, 55, 55, 51, 55, 55, 55, 57, 56, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523490137Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.52350525Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"[2]R flush_numbers[0..12]: [85, 84, 82, 85, 85, 81, 85, 85, 85, 90, 87, 83]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523518853Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"[2]R generation[0..12]: [55, 54, 52, 55, 55, 51, 55, 55, 55, 57, 56, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523532736Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523546759Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523559852Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523572805Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523586157Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523606372Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"":"downstairs","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523620465Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523633108Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"0","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523646881Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"1","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523660273Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"client":"2","":"downstairs","session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523674307Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523686329Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"94fdbaf6-2ea7-476a-9a73-c954b4bdcf9a is now active with session: 100a22e7-adee-45b3-a6e8-daa0a06cce60","v":0,"name":"crucible","level":30,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 time":"2025-03-17T22:22:17.523701373Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523717856Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420,"session_id":"100a22e7-adee-45b3-a6e8-daa0a06cce60"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523730919Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420} {"msg":"Activated sub_volume 94fdbaf6-2ea7-476a-9a73-c954b4bdcf9a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:17.523742791Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 570 len 16384 data: 5 5 6 6 03/30 Write block 164 len 24576 data: 10 15 12 7 6 6 04/30 Read block 419 len 40960 05/30 Write block 440 len 16384 data: 8 7 9 9 06/30 Write block 58 len 36864 data: 6 5 5 4 5 5 5 4 5 07/30 Write block 58 len 36864 data: 7 6 6 5 6 6 6 5 6 08/30 Read block 510 len 20480 09/30 Read block 104 len 24576 10/30 Read block 351 len 36864 11/30 Read block 97 len 12288 12/30 Read block 357 len 8192 13/30 Read block 34 len 8192 14/30 Read block 272 len 12288 15/30 Read block 331 len 28672 16/30 Read block 232 len 28672 17/30 Write block 224 len 40960 data: 7 5 6 7 7 7 8 5 5 6 18/30 Write block 150 len 20480 data: 4 6 5 5 7 19/30 Read block 419 len 12288 20/30 Write block 316 len 8192 data: 7 7 21/30 Write block 246 len 36864 data: 4 4 3 5 7 8 7 7 6 22/30 Write block 169 len 4096 data: 7 23/30 Read block 218 len 20480 24/30 Read block 182 len 40960 25/30 Read block 451 len 36864 26/30 Write block 252 len 28672 data: 8 8 7 5 4 5 5 27/30 Read block 555 len 12288 28/30 Read block 120 len 24576 29/30 Read block 276 len 24576 30/30 Read block 117 len 12288 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 4 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 29 28 29 86 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:17.544308069Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1420} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 85 85 85 F F F 1 020-039 54 54 54 84 84 84 F F F 2 040-059 52 52 52 82 82 82 T T T 3 060-079 55 55 55 85 85 85 T T T 4 080-099 55 55 55 85 85 85 F F F 5 100-119 51 51 51 81 81 81 F F F 6 120-139 55 55 55 85 85 85 F F F 7 140-159 55 55 55 85 85 85 T T T 8 160-179 55 55 55 85 85 85 T T T 9 180-199 57 57 57 90 90 90 F F F 10 200-219 56 56 56 87 87 87 F F F 11 220-239 54 54 54 83 83 83 T T T 12 240-259 55 55 55 85 85 85 T T T 13 260-279 55 55 55 85 85 85 F F F 14 280-299 56 56 56 88 88 88 F F F 15 300-319 52 52 52 82 82 82 T T T 16 320-339 56 56 56 88 88 88 F F F 17 340-359 56 56 56 88 88 88 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 52 52 52 82 82 82 F F F 20 400-419 56 56 56 88 88 88 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 56 56 56 88 88 88 T T T 23 460-479 55 55 55 85 85 85 F F F 24 480-499 55 55 55 85 85 85 F F F 25 500-519 52 52 52 82 82 82 F F F 26 520-539 56 56 56 89 89 89 F F F 27 540-559 56 56 56 89 89 89 F F F 28 560-579 57 57 57 90 90 90 T T T 29 580-599 55 55 55 85 85 85 F F F Max gen: 57, Max flush: 90 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 85 85 85 F F F 1 020-039 54 54 54 84 84 84 F F F 2 040-059 58 58 58 91 91 91 F F F 3 060-079 58 58 58 91 91 91 F F F 4 080-099 55 55 55 85 85 85 F F F 5 100-119 51 51 51 81 81 81 F F F 6 120-139 55 55 55 85 85 85 F F F 7 140-159 58 58 58 91 91 91 F F F 8 160-179 58 58 58 91 91 91 F F F 9 180-199 57 57 57 90 90 90 F F F 10 200-219 56 56 56 87 87 87 F F F 11 220-239 58 58 58 91 91 91 F F F 12 240-259 58 58 58 91 91 91 F F F 13 260-279 55 55 55 85 85 85 F F F 14 280-299 56 56 56 88 88 88 F F F 15 300-319 58 58 58 91 91 91 F F F 16 320-339 56 56 56 88 88 88 F F F 17 340-359 56 56 56 88 88 88 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 52 52 52 82 82 82 F F F 20 400-419 56 56 56 88 88 88 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 58 58 58 91 91 91 F F F 23 460-479 55 55 55 85 85 85 F F F 24 480-499 55 55 55 85 85 85 F F F 25 500-519 52 52 52 82 82 82 F F F 26 520-539 56 56 56 89 89 89 F F F 27 540-559 56 56 56 89 89 89 F F F 28 560-579 58 58 58 91 91 91 F F F 29 580-599 55 55 55 85 85 85 F F F Max gen: 58, Max flush: 91 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.806252559Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:21.848882187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.849245715Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.88078457Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.88147933Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.881514528Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.881528341Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.881542614Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428} {"msg":"Crucible 28e6dad4-6e90-4880-a0d2-dc6b3cd8d0ed has session id: 71b1bbb7-4827-4eb1-8a99-d6678e09a09c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882133881Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Upstairs opts: Upstairs UUID: 28e6dad4-6e90-4880-a0d2-dc6b3cd8d0ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882215639Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Crucible stats registered with UUID: 28e6dad4-6e90-4880-a0d2-dc6b3cd8d0ed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882244015Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:21.882258048Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882698163Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882724409Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428} {"msg":"28e6dad4-6e90-4880-a0d2-dc6b3cd8d0ed active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882751505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882768329Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"0","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882805266Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"0","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882831952Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"1","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882859788Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"1","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882887904Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"2","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882916821Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"2","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882940636Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"io task","client":"2","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882965071Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"io task","client":"0","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.882991337Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"io task","client":"1","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35336)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.883048949Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"io task","client":"2","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34346)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.883080786Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"io task","client":"1","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42889)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.88323777Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"io task","client":"0","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.883651099Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"2","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.883690737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"1","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.883869936Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"0","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.884372224Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"2","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.884405522Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"1","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.884432858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"0","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.884969353Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885069575Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"[0]R flush_numbers[0..12]: [85, 84, 91, 91, 85, 81, 85, 91, 91, 90, 87, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.88513812Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"[0]R generation[0..12]: [55, 54, 58, 58, 55, 51, 55, 58, 58, 57, 56, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885157824Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885172507Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"[1]R flush_numbers[0..12]: [85, 84, 91, 91, 85, 81, 85, 91, 91, 90, 87, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.88518595Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"[1]R generation[0..12]: [55, 54, 58, 58, 55, 51, 55, 58, 58, 57, 56, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885199203Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885212706Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"[2]R flush_numbers[0..12]: [85, 84, 91, 91, 85, 81, 85, 91, 91, 90, 87, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885225869Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"[2]R generation[0..12]: [55, 54, 58, 58, 55, 51, 55, 58, 58, 57, 56, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885243083Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885257255Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Max found gen is 59","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885270188Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Generation requested: 60 >= found:59","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885283071Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885295544Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885314948Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"":"downstairs","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:22:21.885337343Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885363018Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"0","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885388584Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"1","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885406018Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"client":"2","":"downstairs","session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.88541937Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885432353Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"28e6dad4-6e90-4880-a0d2-dc6b3cd8d0ed is now active with session: 71b1bbb7-4827-4eb1-8a99-d6678e09a09c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885454708Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885477513Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428,"session_id":"71b1bbb7-4827-4eb1-8a99-d6678e09a09c"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885500658Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428} {"msg":"Activated sub_volume 28e6dad4-6e90-4880-a0d2-dc6b3cd8d0ed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:21.885514011Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 507 len 16384 data: 5 6 7 7 02/30 Read block 352 len 36864 03/30 Write block 571 len 4096 data: 6 04/30 Read block 2 len 12288 05/30 Write block 583 len 28672 data: 5 7 7 5 4 4 6 06/30 Read block 385 len 32768 07/30 Read block 165 len 20480 08/30 Flush 09/30 Write block 38 len 40960 data: 9 10 7 7 7 7 8 5 6 6 10/30 Write block 28 len 4096 data: 4 11/30 Write block 74 len 28672 data: 12 11 9 7 7 4 4 12/30 Write block 159 len 8192 data: 7 7 13/30 Read block 146 len 40960 14/30 Write block 381 len 40960 data: 5 3 5 4 4 4 5 5 4 5 15/30 Read block 443 len 28672 16/30 Flush 17/30 Write block 67 len 20480 data: 6 6 6 8 7 18/30 Write block 258 len 4096 data: 6 19/30 Write block 171 len 8192 data: 4 5 20/30 Read block 199 len 28672 21/30 Read block 279 len 40960 22/30 Flush 23/30 Write block 562 len 32768 data: 9 8 7 6 4 6 6 6 24/30 Write block 28 len 8192 data: 5 5 25/30 Write block 123 len 40960 data: 7 5 4 3 3 3 3 3 4 4 26/30 Write block 476 len 32768 data: 5 4 4 4 4 4 5 6 27/30 Write block 290 len 4096 data: 7 28/30 Read block 545 len 8192 29/30 Write block 48 len 40960 data: 8 8 8 7 9 9 8 7 7 7 30/30 Read block 162 len 20480 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:21.904970099Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1428} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 85 85 85 F F F 1 020-039 60 60 60 93 93 93 T T T 2 040-059 60 60 60 93 93 93 T T T 3 060-079 60 60 60 94 94 94 F F F 4 080-099 60 60 60 93 93 93 F F F 5 100-119 51 51 51 81 81 81 F F F 6 120-139 55 55 55 85 85 85 T T T 7 140-159 60 60 60 93 93 93 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 57 57 57 90 90 90 F F F 10 200-219 56 56 56 87 87 87 F F F 11 220-239 58 58 58 91 91 91 F F F 12 240-259 60 60 60 94 94 94 F F F 13 260-279 55 55 55 85 85 85 F F F 14 280-299 56 56 56 88 88 88 T T T 15 300-319 58 58 58 91 91 91 F F F 16 320-339 56 56 56 88 88 88 F F F 17 340-359 56 56 56 88 88 88 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 60 60 60 93 93 93 F F F 20 400-419 56 56 56 88 88 88 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 58 58 58 91 91 91 F F F 23 460-479 55 55 55 85 85 85 T T T 24 480-499 55 55 55 85 85 85 T T T 25 500-519 60 60 60 92 92 92 F F F 26 520-539 56 56 56 89 89 89 F F F 27 540-559 56 56 56 89 89 89 F F F 28 560-579 60 60 60 92 92 92 T T T 29 580-599 60 60 60 92 92 92 F F F Max gen: 60, Max flush: 94 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 85 85 85 F F F 1 020-039 61 61 61 95 95 95 F F F 2 040-059 61 61 61 95 95 95 F F F 3 060-079 60 60 60 94 94 94 F F F 4 080-099 60 60 60 93 93 93 F F F 5 100-119 51 51 51 81 81 81 F F F 6 120-139 61 61 61 95 95 95 F F F 7 140-159 60 60 60 93 93 93 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 57 57 57 90 90 90 F F F 10 200-219 56 56 56 87 87 87 F F F 11 220-239 58 58 58 91 91 91 F F F 12 240-259 60 60 60 94 94 94 F F F 13 260-279 55 55 55 85 85 85 F F F 14 280-299 61 61 61 95 95 95 F F F 15 300-319 58 58 58 91 91 91 F F F 16 320-339 56 56 56 88 88 88 F F F 17 340-359 56 56 56 88 88 88 F F F 18 360-379 52 52 52 82 82 82 F F F 19 380-399 60 60 60 93 93 93 F F F 20 400-419 56 56 56 88 88 88 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 58 58 58 91 91 91 F F F 23 460-479 61 61 61 95 95 95 F F F 24 480-499 61 61 61 95 95 95 F F F 25 500-519 60 60 60 92 92 92 F F F 26 520-539 56 56 56 89 89 89 F F F 27 540-559 56 56 56 89 89 89 F F F 28 560-579 61 61 61 95 95 95 F F F 29 580-599 60 60 60 92 92 92 F F F Max gen: 61, Max flush: 95 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.184238137Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:26.227519296Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.227895277Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.259046108Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.25974677Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.259777066Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.259790139Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.259808943Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436} {"msg":"Crucible 7eb64b55-be51-47ec-bd7d-e280eab47e70 has session id: fbbfc8d4-6d39-41de-9499-019d7a8b49af","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.26044409Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Upstairs opts: Upstairs UUID: 7eb64b55-be51-47ec-bd7d-e280eab47e70, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.260479088Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Crucible stats registered with UUID: 7eb64b55-be51-47ec-bd7d-e280eab47e70","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.260495201Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:26.260508634Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.260878904Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.260911991Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436} {"msg":"7eb64b55-be51-47ec-bd7d-e280eab47e70 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.260966393Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.260992708Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"0","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.261021365Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"0","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.261035588Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"1","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.261049901Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"1","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.261062764Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"2","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.261076366Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"2","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.26109368Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.261108724Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.261122367Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"ds_connection connected from Ok(127.0.0.1:54346)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.261146622Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"ds_connection connected from Ok(127.0.0.1:40894)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.261172567Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"ds_connection connected from Ok(127.0.0.1:59867)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.261435364Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.261947915Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"1","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.261991064Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"2","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.262167662Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"0","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.262630582Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"1","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.262659178Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"2","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.262674722Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"0","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263188292Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263335254Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"[0]R flush_numbers[0..12]: [85, 95, 95, 94, 93, 81, 95, 93, 94, 90, 87, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263379574Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"[0]R generation[0..12]: [55, 61, 61, 60, 60, 51, 61, 60, 60, 57, 56, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263401128Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263416731Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"[1]R flush_numbers[0..12]: [85, 95, 95, 94, 93, 81, 95, 93, 94, 90, 87, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263430114Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"[1]R generation[0..12]: [55, 61, 61, 60, 60, 51, 61, 60, 60, 57, 56, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263443467Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.26345724Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"[2]R flush_numbers[0..12]: [85, 95, 95, 94, 93, 81, 95, 93, 94, 90, 87, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263470563Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"[2]R generation[0..12]: [55, 61, 61, 60, 60, 51, 61, 60, 60, 57, 56, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263483776Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263497319Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263509982Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263522805Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263535407Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263548Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"":"downstairs","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263567244Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263579247Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"Wait for a query_work_queue command to finish before sending IO 0","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263600991Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"1","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263614784Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"client":"2","":"downstairs","session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263627647Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.26363942Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"7eb64b55-be51-47ec-bd7d-e280eab47e70 is now active with session: fbbfc8d4-6d39-41de-9499-019d7a8b49af","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263651902Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263663505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436,"session_id":"fbbfc8d4-6d39-41de-9499-019d7a8b49af"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.263675387Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436} {"msg":"Activated sub_volume 7eb64b55-be51-47ec-bd7d-e280eab47e70","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:26.26368643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 569 len 12288 data: 7 6 7 02/30 Read block 41 len 28672 03/30 Read block 344 len 16384 04/30 Read block 260 len 20480 05/30 Read block 493 len 36864 06/30 Read block 420 len 28672 07/30 Read block 419 len 16384 08/30 Write block 350 len 36864 data: 4 4 4 5 5 5 5 5 6 09/30 Read block 305 len 4096 10/30 Write block 465 len 8192 data: 6 5 11/30 Read block 497 len 24576 12/30 Write block 331 len 24576 data: 7 7 8 9 10 8 13/30 Read block 332 len 20480 14/30 Write block 546 len 40960 data: 4 4 4 3 3 3 4 4 5 5 15/30 Write block 554 len 16384 data: 6 6 6 6 16/30 Read block 179 len 4096 17/30 Read block 104 len 8192 18/30 Write block 256 len 4096 data: 5 19/30 Flush 20/30 Write block 196 len 28672 data: 3 4 4 4 7 7 5 21/30 Write block 154 len 8192 data: 8 8 22/30 Write block 15 len 8192 data: 2 3 23/30 Read block 429 len 4096 24/30 Read block 339 len 24576 25/30 Write block 216 len 4096 data: 3 26/30 Read block 359 len 40960 27/30 Write block 378 len 20480 data: 5 6 5 6 4 28/30 Write block 142 len 32768 data: 4 4 4 4 3 4 5 5 29/30 Read block 76 len 20480 30/30 Read block 464 len 20480 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 7 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 11 10 11 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:26.28329106Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1436} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 85 85 85 T T T 1 020-039 61 61 61 95 95 95 F F F 2 040-059 61 61 61 95 95 95 F F F 3 060-079 60 60 60 94 94 94 F F F 4 080-099 60 60 60 93 93 93 F F F 5 100-119 51 51 51 81 81 81 F F F 6 120-139 61 61 61 95 95 95 F F F 7 140-159 60 60 60 93 93 93 T T T 8 160-179 60 60 60 94 94 94 F F F 9 180-199 57 57 57 90 90 90 T T T 10 200-219 56 56 56 87 87 87 T T T 11 220-239 58 58 58 91 91 91 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 55 55 55 85 85 85 F F F 14 280-299 61 61 61 95 95 95 F F F 15 300-319 58 58 58 91 91 91 F F F 16 320-339 62 62 62 96 96 96 F F F 17 340-359 62 62 62 96 96 96 F F F 18 360-379 52 52 52 82 82 82 T T T 19 380-399 60 60 60 93 93 93 T T T 20 400-419 56 56 56 88 88 88 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 58 58 58 91 91 91 F F F 23 460-479 62 62 62 96 96 96 F F F 24 480-499 61 61 61 95 95 95 F F F 25 500-519 60 60 60 92 92 92 F F F 26 520-539 56 56 56 89 89 89 F F F 27 540-559 62 62 62 96 96 96 F F F 28 560-579 62 62 62 96 96 96 F F F 29 580-599 60 60 60 92 92 92 F F F Max gen: 62, Max flush: 96 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 97 97 97 F F F 1 020-039 61 61 61 95 95 95 F F F 2 040-059 61 61 61 95 95 95 F F F 3 060-079 60 60 60 94 94 94 F F F 4 080-099 60 60 60 93 93 93 F F F 5 100-119 51 51 51 81 81 81 F F F 6 120-139 61 61 61 95 95 95 F F F 7 140-159 63 63 63 97 97 97 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 63 63 63 97 97 97 F F F 10 200-219 63 63 63 97 97 97 F F F 11 220-239 58 58 58 91 91 91 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 55 55 55 85 85 85 F F F 14 280-299 61 61 61 95 95 95 F F F 15 300-319 58 58 58 91 91 91 F F F 16 320-339 62 62 62 96 96 96 F F F 17 340-359 62 62 62 96 96 96 F F F 18 360-379 63 63 63 97 97 97 F F F 19 380-399 63 63 63 97 97 97 F F F 20 400-419 56 56 56 88 88 88 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 58 58 58 91 91 91 F F F 23 460-479 62 62 62 96 96 96 F F F 24 480-499 61 61 61 95 95 95 F F F 25 500-519 60 60 60 92 92 92 F F F 26 520-539 56 56 56 89 89 89 F F F 27 540-559 62 62 62 96 96 96 F F F 28 560-579 62 62 62 96 96 96 F F F 29 580-599 60 60 60 92 92 92 F F F Max gen: 63, Max flush: 97 Begin loop 32 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.554544648Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:30.597086088Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.597477822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.628587005Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.629221662Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.62925792Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.629272043Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.629286706Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444} {"msg":"Crucible 13b269fb-1cc5-4e02-b64f-1acd7f1f447a has session id: ee6b7284-1635-4caf-8a61-730c7a5370d8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.629941657Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Upstairs opts: Upstairs UUID: 13b269fb-1cc5-4e02-b64f-1acd7f1f447a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.629981586Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Crucible stats registered with UUID: 13b269fb-1cc5-4e02-b64f-1acd7f1f447a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.629998279Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:30.630011632Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630390844Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630413059Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444} {"msg":"13b269fb-1cc5-4e02-b64f-1acd7f1f447a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630477263Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630495527Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"0","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630523042Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"0","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630536685Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"1","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630550488Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"1","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630563301Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"2","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630577064Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"2","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630589887Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"io task","client":"2","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.63060322Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"io task","client":"0","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630616402Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"io task","client":"1","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:55919)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630770366Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"io task","client":"0","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:61248)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630813115Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"io task","client":"2","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:63981)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.630912297Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"io task","client":"1","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.631570459Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"2","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.631598814Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"1","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.631698886Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"0","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.632321Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"2","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.632355058Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"1","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.632380613Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"0","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.632926711Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633034254Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"[0]R flush_numbers[0..12]: [97, 95, 95, 94, 93, 81, 95, 97, 94, 97, 97, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633055889Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"[0]R generation[0..12]: [63, 61, 61, 60, 60, 51, 61, 63, 60, 63, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633071472Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633086115Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"[1]R flush_numbers[0..12]: [97, 95, 95, 94, 93, 81, 95, 97, 94, 97, 97, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633099818Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"[1]R generation[0..12]: [63, 61, 61, 60, 60, 51, 61, 63, 60, 63, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633113471Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633127124Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"[2]R flush_numbers[0..12]: [97, 95, 95, 94, 93, 81, 95, 97, 94, 97, 97, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633140377Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"[2]R generation[0..12]: [63, 61, 61, 60, 60, 51, 61, 63, 60, 63, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.63315352Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633170274Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633191368Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633215914Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633239179Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"All extents match","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-03-17T22:22:30.633266074Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"":"downstairs","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.63329081Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633307773Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"0","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633321476Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"1","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633334419Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"client":"2","":"downstairs","session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633347272Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633359054Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"13b269fb-1cc5-4e02-b64f-1acd7f1f447a is now active with session: ee6b7284-1635-4caf-8a61-730c7a5370d8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633371487Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.63338311Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444,"session_id":"ee6b7284-1635-4caf-8a61-730c7a5370d8"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633395002Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444} {"msg":"Activated sub_volume 13b269fb-1cc5-4e02-b64f-1acd7f1f447a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:30.633405935Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 463 len 12288 data: 7 6 7 02/30 Read block 545 len 32768 03/30 Write block 324 len 32768 data: 5 6 6 6 4 4 5 8 04/30 Write block 524 len 12288 data: 7 7 9 05/30 Flush 06/30 Flush 07/30 Write block 556 len 16384 data: 7 7 9 8 08/30 Write block 18 len 4096 data: 3 09/30 Read block 461 len 12288 10/30 Read block 242 len 32768 11/30 Write block 416 len 16384 data: 4 4 4 4 12/30 Read block 467 len 32768 13/30 Flush 14/30 Write block 302 len 4096 data: 5 15/30 Write block 495 len 8192 data: 8 7 16/30 Write block 411 len 16384 data: 4 4 3 3 17/30 Write block 549 len 20480 data: 4 4 4 5 5 18/30 Read block 228 len 32768 19/30 Read block 2 len 28672 20/30 Write block 137 len 16384 data: 3 5 6 4 21/30 Read block 573 len 36864 22/30 Write block 308 len 4096 data: 4 23/30 Write block 587 len 32768 data: 5 5 7 6 6 5 3 2 24/30 Flush 25/30 Write block 344 len 8192 data: 2 2 26/30 Flush 27/30 Write block 52 len 16384 data: 10 10 9 8 28/30 Read block 135 len 16384 29/30 Write block 64 len 28672 data: 7 6 7 7 7 7 9 30/30 Write block 115 len 16384 data: 5 7 6 7 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:30.651462091Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1444} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 100 100 100 F F F 1 020-039 61 61 61 95 95 95 F F F 2 040-059 61 61 61 95 95 95 T T T 3 060-079 60 60 60 94 94 94 T T T 4 080-099 60 60 60 93 93 93 F F F 5 100-119 51 51 51 81 81 81 T T T 6 120-139 64 64 64 101 101 101 F F F 7 140-159 64 64 64 101 101 101 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 63 63 63 97 97 97 F F F 10 200-219 63 63 63 97 97 97 F F F 11 220-239 58 58 58 91 91 91 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 55 55 55 85 85 85 F F F 14 280-299 61 61 61 95 95 95 F F F 15 300-319 64 64 64 101 101 101 F F F 16 320-339 64 64 64 98 98 98 F F F 17 340-359 64 64 64 102 102 102 F F F 18 360-379 63 63 63 97 97 97 F F F 19 380-399 63 63 63 97 97 97 F F F 20 400-419 64 64 64 101 101 101 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 58 58 58 91 91 91 F F F 23 460-479 64 64 64 98 98 98 F F F 24 480-499 64 64 64 101 101 101 F F F 25 500-519 60 60 60 92 92 92 F F F 26 520-539 64 64 64 98 98 98 F F F 27 540-559 64 64 64 101 101 101 F F F 28 560-579 62 62 62 96 96 96 F F F 29 580-599 64 64 64 101 101 101 F F F Max gen: 64, Max flush: 102 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 100 100 100 F F F 1 020-039 61 61 61 95 95 95 F F F 2 040-059 65 65 65 103 103 103 F F F 3 060-079 65 65 65 103 103 103 F F F 4 080-099 60 60 60 93 93 93 F F F 5 100-119 65 65 65 103 103 103 F F F 6 120-139 64 64 64 101 101 101 F F F 7 140-159 64 64 64 101 101 101 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 63 63 63 97 97 97 F F F 10 200-219 63 63 63 97 97 97 F F F 11 220-239 58 58 58 91 91 91 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 55 55 55 85 85 85 F F F 14 280-299 61 61 61 95 95 95 F F F 15 300-319 64 64 64 101 101 101 F F F 16 320-339 64 64 64 98 98 98 F F F 17 340-359 64 64 64 102 102 102 F F F 18 360-379 63 63 63 97 97 97 F F F 19 380-399 63 63 63 97 97 97 F F F 20 400-419 64 64 64 101 101 101 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 58 58 58 91 91 91 F F F 23 460-479 64 64 64 98 98 98 F F F 24 480-499 64 64 64 101 101 101 F F F 25 500-519 60 60 60 92 92 92 F F F 26 520-539 64 64 64 98 98 98 F F F 27 540-559 64 64 64 101 101 101 F F F 28 560-579 62 62 62 96 96 96 F F F 29 580-599 64 64 64 101 101 101 F F F Max gen: 65, Max flush: 103 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.906690591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.95053047Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.951281793Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.951323982Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.951338715Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.951352948Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452} {"msg":"Crucible 217cd84d-36a2-438b-9bd2-2cda8b2d4b3b has session id: 071eeb1b-3d5b-425d-9b73-97f8ee7da20d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952053839Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Upstairs opts: Upstairs UUID: 217cd84d-36a2-438b-9bd2-2cda8b2d4b3b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952094708Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Crucible stats registered with UUID: 217cd84d-36a2-438b-9bd2-2cda8b2d4b3b","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952110991Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:34.952123614Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952551286Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952584254Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452} {"msg":"217cd84d-36a2-438b-9bd2-2cda8b2d4b3b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952638595Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952665201Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"0","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952696618Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"0","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952712451Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"1","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952737337Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"1","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952759442Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"2","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952777085Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"2","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.95279929Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"io task","client":"2","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952822185Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"io task","client":"0","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.95284508Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54659)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.952904023Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"io task","client":"0","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:34.953065187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:34.953100855Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:34.953118259Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46754)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.953130562Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"io task","client":"2","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.953250557Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.9536345Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"2","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.953670688Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"0","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.95432646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"0","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.95446799Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"2","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.954909305Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:34.955005566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.954782709Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54331)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.955268894Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.95622254Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"1","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.956764837Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"1","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"[0]R flush_numbers[0..12]: [100, 95, 103, 103, 93, 103, 101, 101, 94, 97, 97, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.95738136Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"[0]R generation[0..12]: [64, 61, 65, 65, 60, 65, 64, 64, 60, 63, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957405055Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957422869Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"[1]R flush_numbers[0..12]: [100, 95, 103, 103, 93, 103, 101, 101, 94, 97, 97, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957439942Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"[1]R generation[0..12]: [64, 61, 65, 65, 60, 65, 64, 64, 60, 63, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957456456Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.95747354Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"[2]R flush_numbers[0..12]: [100, 95, 103, 103, 93, 103, 101, 101, 94, 97, 97, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957490533Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"[2]R generation[0..12]: [64, 61, 65, 65, 60, 65, 64, 64, 60, 63, 63, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957507227Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957524351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957539754Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957555518Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957571381Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957593956Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957618241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957632804Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"0","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957648908Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"1","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957664671Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"client":"2","":"downstairs","session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957680405Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957695128Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"217cd84d-36a2-438b-9bd2-2cda8b2d4b3b is now active with session: 071eeb1b-3d5b-425d-9b73-97f8ee7da20d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957710881Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957725714Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452,"session_id":"071eeb1b-3d5b-425d-9b73-97f8ee7da20d"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.957740427Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452} {"msg":"Activated sub_volume 217cd84d-36a2-438b-9bd2-2cda8b2d4b3b","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:44.95775397Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 302 len 16384 data: 6 4 4 4 02/30 Flush 03/30 Write block 26 len 40960 data: 4 4 6 6 4 4 4 5 4 5 04/30 Read block 314 len 28672 05/30 Read block 483 len 36864 06/30 Write block 549 len 20480 data: 5 5 5 6 6 07/30 Write block 560 len 36864 data: 10 9 10 9 8 7 5 7 7 08/30 Write block 591 len 4096 data: 7 09/30 Write block 147 len 24576 data: 5 6 6 5 7 6 10/30 Write block 294 len 20480 data: 5 5 8 8 9 11/30 Write block 512 len 32768 data: 7 7 8 9 8 9 9 7 12/30 Read block 372 len 36864 13/30 Flush 14/30 Write block 464 len 20480 data: 7 8 6 10 11 15/30 Read block 106 len 24576 16/30 Read block 475 len 16384 17/30 Read block 204 len 40960 18/30 Write block 451 len 40960 data: 6 7 8 7 5 3 3 3 3 4 19/30 Write block 218 len 8192 data: 5 6 20/30 Read block 583 len 12288 21/30 Read block 107 len 8192 22/30 Read block 3 len 20480 23/30 Read block 330 len 28672 24/30 Write block 195 len 12288 data: 5 4 5 25/30 Read block 323 len 4096 26/30 Read block 392 len 8192 27/30 Write block 80 len 16384 data: 5 4 3 5 28/30 Write block 185 len 20480 data: 11 9 8 8 8 29/30 Write block 85 len 36864 data: 5 6 6 6 5 7 6 5 5 30/30 Read block 121 len 40960 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 5 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 16 15 17 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:44.978024994Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1452} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 100 100 100 F F F 1 020-039 66 66 66 105 105 105 F F F 2 040-059 65 65 65 103 103 103 F F F 3 060-079 65 65 65 103 103 103 F F F 4 080-099 60 60 60 93 93 93 T T T 5 100-119 65 65 65 103 103 103 F F F 6 120-139 64 64 64 101 101 101 F F F 7 140-159 66 66 66 105 105 105 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 63 63 63 97 97 97 T T T 10 200-219 63 63 63 97 97 97 T T T 11 220-239 58 58 58 91 91 91 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 55 55 55 85 85 85 F F F 14 280-299 66 66 66 105 105 105 F F F 15 300-319 66 66 66 104 104 104 F F F 16 320-339 64 64 64 98 98 98 F F F 17 340-359 64 64 64 102 102 102 F F F 18 360-379 63 63 63 97 97 97 F F F 19 380-399 63 63 63 97 97 97 F F F 20 400-419 64 64 64 101 101 101 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 58 58 58 91 91 91 T T T 23 460-479 64 64 64 98 98 98 T T T 24 480-499 64 64 64 101 101 101 F F F 25 500-519 66 66 66 105 105 105 F F F 26 520-539 64 64 64 98 98 98 F F F 27 540-559 66 66 66 105 105 105 F F F 28 560-579 66 66 66 105 105 105 F F F 29 580-599 66 66 66 105 105 105 F F F Max gen: 66, Max flush: 105 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 100 100 100 F F F 1 020-039 66 66 66 105 105 105 F F F 2 040-059 65 65 65 103 103 103 F F F 3 060-079 65 65 65 103 103 103 F F F 4 080-099 67 67 67 106 106 106 F F F 5 100-119 65 65 65 103 103 103 F F F 6 120-139 64 64 64 101 101 101 F F F 7 140-159 66 66 66 105 105 105 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 67 67 67 106 106 106 F F F 10 200-219 67 67 67 106 106 106 F F F 11 220-239 58 58 58 91 91 91 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 55 55 55 85 85 85 F F F 14 280-299 66 66 66 105 105 105 F F F 15 300-319 66 66 66 104 104 104 F F F 16 320-339 64 64 64 98 98 98 F F F 17 340-359 64 64 64 102 102 102 F F F 18 360-379 63 63 63 97 97 97 F F F 19 380-399 63 63 63 97 97 97 F F F 20 400-419 64 64 64 101 101 101 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 67 67 67 106 106 106 F F F 23 460-479 67 67 67 106 106 106 F F F 24 480-499 64 64 64 101 101 101 F F F 25 500-519 66 66 66 105 105 105 F F F 26 520-539 64 64 64 98 98 98 F F F 27 540-559 66 66 66 105 105 105 F F F 28 560-579 66 66 66 105 105 105 F F F 29 580-599 66 66 66 105 105 105 F F F Max gen: 67, Max flush: 106 Begin loop 34 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.220599625Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:49.263359822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.263748895Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.294965551Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.295552728Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.295586425Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.295600928Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.295615271Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460} {"msg":"Crucible 4af95084-4288-4a2d-b895-6636b82cf824 has session id: 9995ecdc-b0a4-4362-9b8a-01f89c31775d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.296215971Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Upstairs opts: Upstairs UUID: 4af95084-4288-4a2d-b895-6636b82cf824, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.296266642Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Crucible stats registered with UUID: 4af95084-4288-4a2d-b895-6636b82cf824","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.296290117Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:49.29630406Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.296735083Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.29676756Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460} {"msg":"4af95084-4288-4a2d-b895-6636b82cf824 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.296819231Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.296843987Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"0","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.296888636Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"0","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.296918483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"1","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.296949289Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"1","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.296976145Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"2","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.297003661Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"2","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.297027526Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"io task","client":"2","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.29704814Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"io task","client":"0","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.297074066Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"io task","client":"1","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57362)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.297105103Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"io task","client":"2","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36712)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.297132419Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"io task","client":"1","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57125)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.297189831Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"io task","client":"0","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.297790031Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"2","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.297969109Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"1","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.298022391Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"0","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.298401783Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"2","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.298557546Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"1","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.2986656Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"0","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299019936Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.29913179Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"[0]R flush_numbers[0..12]: [100, 105, 103, 103, 106, 103, 101, 105, 94, 106, 106, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299248205Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"[0]R generation[0..12]: [64, 66, 65, 65, 67, 65, 64, 66, 60, 67, 67, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.2992694Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299284723Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"[1]R flush_numbers[0..12]: [100, 105, 103, 103, 106, 103, 101, 105, 94, 106, 106, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299302737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"[1]R generation[0..12]: [64, 66, 65, 65, 67, 65, 64, 66, 60, 67, 67, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.29931693Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299331143Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"[2]R flush_numbers[0..12]: [100, 105, 103, 103, 106, 103, 101, 105, 94, 106, 106, 91]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299344806Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"[2]R generation[0..12]: [64, 66, 65, 65, 67, 65, 64, 66, 60, 67, 67, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299358109Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299371962Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299384615Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299404469Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Next flush: 107","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-03-17T22:22:49.299434105Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.2994566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"":"downstairs","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299480255Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.29950259Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"0","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299526555Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"1","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.29955058Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"client":"2","":"downstairs","session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299569044Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299588969Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"4af95084-4288-4a2d-b895-6636b82cf824 is now active with session: 9995ecdc-b0a4-4362-9b8a-01f89c31775d","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-03-17T22:22:49.299612184Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299638249Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460,"session_id":"9995ecdc-b0a4-4362-9b8a-01f89c31775d"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299655583Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460} {"msg":"Activated sub_volume 4af95084-4288-4a2d-b895-6636b82cf824","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:49.299667446Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 414 len 12288 data: 4 3 5 02/30 Read block 293 len 4096 03/30 Read block 364 len 36864 04/30 Write block 280 len 12288 data: 4 5 5 05/30 Read block 162 len 12288 06/30 Write block 581 len 36864 data: 6 5 6 8 8 6 6 6 8 07/30 Read block 368 len 4096 08/30 Read block 22 len 12288 09/30 Read block 47 len 32768 10/30 Write block 550 len 28672 data: 6 6 7 7 7 7 8 11/30 Write block 539 len 40960 data: 4 4 3 4 4 4 4 5 5 5 12/30 Flush 13/30 Write block 222 len 28672 data: 5 6 8 6 7 8 8 14/30 Write block 319 len 28672 data: 6 6 5 5 5 6 7 15/30 Write block 235 len 20480 data: 7 6 5 5 5 16/30 Write block 584 len 32768 data: 9 9 7 7 7 9 7 8 17/30 Read block 509 len 36864 18/30 Write block 357 len 16384 data: 6 7 6 8 19/30 Write block 2 len 4096 data: 4 20/30 Read block 124 len 4096 21/30 Read block 295 len 40960 22/30 Write block 43 len 32768 data: 8 9 6 7 7 9 9 9 23/30 Read block 286 len 8192 24/30 Read block 123 len 12288 25/30 Read block 525 len 24576 26/30 Write block 274 len 4096 data: 5 27/30 Read block 139 len 8192 28/30 Write block 84 len 12288 data: 5 6 7 29/30 Write block 384 len 28672 data: 5 5 5 6 6 5 6 30/30 Read block 231 len 32768 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 7 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 7 Done Sent Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 18 16 16 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:49.319711581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1460} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 100 100 100 T T T 1 020-039 66 66 66 105 105 105 F F F 2 040-059 65 65 65 103 103 103 T T T 3 060-079 65 65 65 103 103 103 F F F 4 080-099 67 67 67 106 106 106 T T T 5 100-119 65 65 65 103 103 103 F F F 6 120-139 64 64 64 101 101 101 F F F 7 140-159 66 66 66 105 105 105 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 67 67 67 106 106 106 F F F 10 200-219 67 67 67 106 106 106 F F F 11 220-239 58 58 58 91 91 91 T T T 12 240-259 62 62 62 96 96 96 F F F 13 260-279 55 55 55 85 85 85 T T T 14 280-299 68 68 68 107 107 107 F F F 15 300-319 66 66 66 104 104 104 T T T 16 320-339 64 64 64 98 98 98 T T T 17 340-359 64 64 64 102 102 102 T T T 18 360-379 63 63 63 97 97 97 T T T 19 380-399 63 63 63 97 97 97 T T T 20 400-419 68 68 68 107 107 107 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 67 67 67 106 106 106 F F F 23 460-479 67 67 67 106 106 106 F F F 24 480-499 64 64 64 101 101 101 F F F 25 500-519 66 66 66 105 105 105 F F F 26 520-539 68 68 68 107 107 107 F F F 27 540-559 68 68 68 107 107 107 F F F 28 560-579 66 66 66 105 105 105 F F F 29 580-599 68 68 68 107 107 107 T T T Max gen: 68, Max flush: 107 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 108 108 108 F F F 1 020-039 66 66 66 105 105 105 F F F 2 040-059 69 69 69 108 108 108 F F F 3 060-079 65 65 65 103 103 103 F F F 4 080-099 69 69 69 108 108 108 F F F 5 100-119 65 65 65 103 103 103 F F F 6 120-139 64 64 64 101 101 101 F F F 7 140-159 66 66 66 105 105 105 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 67 67 67 106 106 106 F F F 10 200-219 67 67 67 106 106 106 F F F 11 220-239 69 69 69 108 108 108 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 69 69 69 108 108 108 F F F 14 280-299 68 68 68 107 107 107 F F F 15 300-319 69 69 69 108 108 108 F F F 16 320-339 69 69 69 108 108 108 F F F 17 340-359 69 69 69 108 108 108 F F F 18 360-379 69 69 69 108 108 108 F F F 19 380-399 69 69 69 108 108 108 F F F 20 400-419 68 68 68 107 107 107 F F F 21 420-439 56 56 56 89 89 89 F F F 22 440-459 67 67 67 106 106 106 F F F 23 460-479 67 67 67 106 106 106 F F F 24 480-499 64 64 64 101 101 101 F F F 25 500-519 66 66 66 105 105 105 F F F 26 520-539 68 68 68 107 107 107 F F F 27 540-559 68 68 68 107 107 107 F F F 28 560-579 66 66 66 105 105 105 F F F 29 580-599 69 69 69 108 108 108 F F F Max gen: 69, Max flush: 108 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.622231876Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:53.666620704Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.66711322Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.699731728Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.700641195Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.700685924Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.700703678Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.700721192Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468} {"msg":"Crucible 3a302474-cc91-4f24-9e89-d96108704bdf has session id: c844ecae-ec29-41ad-8884-3833f285ab19","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.701384445Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"Upstairs opts: Upstairs UUID: 3a302474-cc91-4f24-9e89-d96108704bdf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.701427754Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"Crucible stats registered with UUID: 3a302474-cc91-4f24-9e89-d96108704bdf","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.701445228Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:53.701457881Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.701894955Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.70191898Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468} {"msg":"3a302474-cc91-4f24-9e89-d96108704bdf active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.701974382Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.702004819Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"0","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.702043027Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"0","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.702059051Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"1","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.702073904Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"1","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.702087347Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"2","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.70210402Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"2","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.702126345Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"io task","client":"2","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.70215067Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"io task","client":"0","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.702174986Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"io task","client":"1","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"ds_connection connected from Ok(127.0.0.1:35698)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.702198461Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"io task","client":"0","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"ds_connection connected from Ok(127.0.0.1:53108)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.702323067Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"io task","client":"1","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"ds_connection connected from Ok(127.0.0.1:37667)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.702365107Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"io task","client":"2","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.703024929Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"1","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.703064218Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"2","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.703241646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"0","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.703776581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"2","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.70390949Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"1","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.703942037Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"0","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.70432747Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704538245Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"[0]R flush_numbers[0..12]: [108, 105, 108, 103, 108, 103, 101, 105, 94, 106, 106, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704676225Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"[0]R generation[0..12]: [69, 66, 69, 65, 69, 65, 64, 66, 60, 67, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704704011Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704719405Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"[1]R flush_numbers[0..12]: [108, 105, 108, 103, 108, 103, 101, 105, 94, 106, 106, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704733468Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"[1]R generation[0..12]: [69, 66, 69, 65, 69, 65, 64, 66, 60, 67, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704747461Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704761584Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"[2]R flush_numbers[0..12]: [108, 105, 108, 103, 108, 103, 101, 105, 94, 106, 106, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704775057Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"[2]R generation[0..12]: [69, 66, 69, 65, 69, 65, 64, 66, 60, 67, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.70479237Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704806583Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704819236Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704832179Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704852673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704866676Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"":"downstairs","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704879529Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704891132Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"0","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704903995Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"1","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704916797Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"client":"2","":"downstairs","session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.70492965Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {Wait for a query_work_queue command to finish before sending IO "msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704948204Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"3a302474-cc91-4f24-9e89-d96108704bdf is now active with session: c844ecae-ec29-41ad-8884-3833f285ab19","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704960667Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704972229Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468,"session_id":"c844ecae-ec29-41ad-8884-3833f285ab19"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704983802Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468} {"msg":"Activated sub_volume 3a302474-cc91-4f24-9e89-d96108704bdf","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:53.704994614Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 366 len 40960 data: 4 5 4 3 3 3 5 5 5 6 02/30 Read block 119 len 16384 03/30 Read block 473 len 24576 04/30 Read block 392 len 8192 05/30 Flush 06/30 Write block 488 len 20480 data: 7 10 10 10 10 07/30 Read block 479 len 40960 08/30 Read block 373 len 4096 09/30 Flush 10/30 Write block 331 len 16384 data: 9 8 9 10 11/30 Read block 320 len 4096 12/30 Read block 469 len 36864 13/30 Read block 8 len 4096 14/30 Write block 12 len 8192 data: 6 3 15/30 Read block 146 len 28672 16/30 Write block 132 len 12288 data: 5 5 5 17/30 Write block 120 len 16384 data: 6 6 8 8 18/30 Read block 85 len 4096 19/30 Read block 594 len 24576 20/30 Read block 497 len 16384 21/30 Write block 488 len 20480 data: 8 11 11 11 11 22/30 Read block 317 len 8192 23/30 Flush 24/30 Write block 483 len 4096 data: 7 25/30 Read block 386 len 28672 26/30 Write block 393 len 8192 data: 6 5 27/30 Write block 424 len 20480 data: 5 4 5 6 6 28/30 Read block 162 len 32768 29/30 Read block 94 len 16384 30/30 Write block 118 len 40960 data: 8 6 7 7 9 9 6 5 4 4 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:53.725909467Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1468} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 111 111 111 F F F 1 020-039 66 66 66 105 105 105 F F F 2 040-059 69 69 69 108 108 108 F F F 3 060-079 65 65 65 103 103 103 F F F 4 080-099 69 69 69 108 108 108 F F F 5 100-119 65 65 65 103 103 103 T T T 6 120-139 70 70 70 111 111 111 T T T 7 140-159 66 66 66 105 105 105 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 67 67 67 106 106 106 F F F 10 200-219 67 67 67 106 106 106 F F F 11 220-239 69 69 69 108 108 108 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 69 69 69 108 108 108 F F F 14 280-299 68 68 68 107 107 107 F F F 15 300-319 69 69 69 108 108 108 F F F 16 320-339 70 70 70 111 111 111 F F F 17 340-359 69 69 69 108 108 108 F F F 18 360-379 70 70 70 109 109 109 F F F 19 380-399 69 69 69 108 108 108 T T T 20 400-419 68 68 68 107 107 107 F F F 21 420-439 56 56 56 89 89 89 T T T 22 440-459 67 67 67 106 106 106 F F F 23 460-479 67 67 67 106 106 106 F F F 24 480-499 70 70 70 111 111 111 T T T 25 500-519 66 66 66 105 105 105 F F F 26 520-539 68 68 68 107 107 107 F F F 27 540-559 68 68 68 107 107 107 F F F 28 560-579 66 66 66 105 105 105 F F F 29 580-599 69 69 69 108 108 108 F F F Max gen: 70, Max flush: 111 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 111 111 111 F F F 1 020-039 66 66 66 105 105 105 F F F 2 040-059 69 69 69 108 108 108 F F F 3 060-079 65 65 65 103 103 103 F F F 4 080-099 69 69 69 108 108 108 F F F 5 100-119 71 71 71 112 112 112 F F F 6 120-139 71 71 71 112 112 112 F F F 7 140-159 66 66 66 105 105 105 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 67 67 67 106 106 106 F F F 10 200-219 67 67 67 106 106 106 F F F 11 220-239 69 69 69 108 108 108 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 69 69 69 108 108 108 F F F 14 280-299 68 68 68 107 107 107 F F F 15 300-319 69 69 69 108 108 108 F F F 16 320-339 70 70 70 111 111 111 F F F 17 340-359 69 69 69 108 108 108 F F F 18 360-379 70 70 70 109 109 109 F F F 19 380-399 71 71 71 112 112 112 F F F 20 400-419 68 68 68 107 107 107 F F F 21 420-439 71 71 71 112 112 112 F F F 22 440-459 67 67 67 106 106 106 F F F 23 460-479 67 67 67 106 106 106 F F F 24 480-499 71 71 71 112 112 112 F F F 25 500-519 66 66 66 105 105 105 F F F 26 520-539 68 68 68 107 107 107 F F F 27 540-559 68 68 68 107 107 107 F F F 28 560-579 66 66 66 105 105 105 F F F 29 580-599 69 69 69 108 108 108 F F F Max gen: 71, Max flush: 112 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:57.999833251Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.044409499Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.04524635Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.045289809Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.045304702Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.045320276Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476} {"msg":"Crucible 6f0c1614-395f-4b91-8181-435c2f372983 has session id: d112c8a6-e2bc-429d-a4db-58132be17bab","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046003193Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Upstairs opts: Upstairs UUID: 6f0c1614-395f-4b91-8181-435c2f372983, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046055234Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Crucible stats registered with UUID: 6f0c1614-395f-4b91-8181-435c2f372983","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046075509Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:58.046088241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046462762Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046488288Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476} {"msg":"6f0c1614-395f-4b91-8181-435c2f372983 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046563644Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046580858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"0","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046608844Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"0","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046628718Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"1","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046654223Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"1","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046678929Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"2","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046706355Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"2","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.04673234Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"io task","client":"2","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046756946Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"io task","client":"0","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046818039Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"io task","client":"1","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"ds_connection connected from Ok(127.0.0.1:56368)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046846915Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"io task","client":"2","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"ds_connection connected from Ok(127.0.0.1:47597)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.046872221Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"io task","client":"0","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:58.047060341Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"io task","client":"1","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:58.047089577Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"io task","client":"1","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:22:58.047114763Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.047134947Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"io task","client":"1","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.047567371Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"2","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.047595466Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"0","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.048277614Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"0","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.04830565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"2","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.048922183Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:22:58.048949749Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.048800778Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"io task","client":"1","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"ds_connection connected from Ok(127.0.0.1:35356)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.049064565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"io task","client":"1","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.050041246Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"1","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.050604057Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"1","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"[0]R flush_numbers[0..12]: [111, 105, 108, 103, 108, 112, 112, 105, 94, 106, 106, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051274672Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"[0]R generation[0..12]: [70, 66, 69, 65, 69, 71, 71, 66, 60, 67, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051300167Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051315891Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"[1]R flush_numbers[0..12]: [111, 105, 108, 103, 108, 112, 112, 105, 94, 106, 106, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051329924Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"[1]R generation[0..12]: [70, 66, 69, 65, 69, 71, 71, 66, 60, 67, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051343707Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.05135758Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"[2]R flush_numbers[0..12]: [111, 105, 108, 103, 108, 112, 112, 105, 94, 106, 106, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051371273Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"[2]R generation[0..12]: [70, 66, 69, 65, 69, 71, 71, 66, 60, 67, 67, 69]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051384435Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051398208Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051411031Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051423974Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051441018Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051455991Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"":"downstairs","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051472024Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051486418Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"0","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051502751Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"1","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051518664Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"client":"2","":"downstairs","session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051534348Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051549221Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"6f0c1614-395f-4b91-8181-435c2f372983 is now active with session: d112c8a6-e2bc-429d-a4db-58132be17bab","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:23:08.051565024Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051584949Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476,"session_id":"d112c8a6-e2bc-429d-a4db-58132be17bab"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051599882Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476} {"msg":"Activated sub_volume 6f0c1614-395f-4b91-8181-435c2f372983","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:08.051613615Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 273 len 16384 data: 6 6 6 6 03/30 Read block 386 len 12288 04/30 Read block 101 len 36864 05/30 Write block 283 len 28672 data: 6 7 8 7 5 5 4 06/30 Write block 467 len 12288 data: 11 12 10 07/30 Flush 08/30 Write block 230 len 8192 data: 9 6 09/30 Read block 73 len 28672 10/30 Write block 370 len 8192 data: 4 4 11/30 Read block 109 len 16384 12/30 Write block 488 len 28672 data: 9 12 12 12 12 10 8 13/30 Read block 346 len 8192 14/30 Write block 497 len 28672 data: 7 7 8 8 10 9 10 15/30 Write block 209 len 4096 data: 5 16/30 Write block 182 len 16384 data: 11 11 13 12 17/30 Write block 334 len 12288 data: 11 11 9 18/30 Write block 348 len 4096 data: 3 19/30 Read block 525 len 16384 20/30 Read block 312 len 20480 21/30 Read block 43 len 12288 22/30 Write block 567 len 28672 data: 8 8 8 7 8 7 7 23/30 Read block 281 len 8192 24/30 Read block 145 len 40960 25/30 Flush 26/30 Write block 42 len 12288 data: 8 9 10 27/30 Read block 379 len 24576 28/30 Write block 114 len 40960 data: 5 6 8 7 9 7 8 8 10 10 29/30 Write block 271 len 12288 data: 6 7 7 30/30 Write block 209 len 12288 data: 6 4 4 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:08.070707495Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1476} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 111 111 111 F F F 1 020-039 66 66 66 105 105 105 F F F 2 040-059 69 69 69 108 108 108 T T T 3 060-079 65 65 65 103 103 103 F F F 4 080-099 69 69 69 108 108 108 F F F 5 100-119 71 71 71 112 112 112 T T T 6 120-139 71 71 71 112 112 112 T T T 7 140-159 66 66 66 105 105 105 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 72 72 72 115 115 115 F F F 10 200-219 72 72 72 115 115 115 T T T 11 220-239 72 72 72 115 115 115 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 72 72 72 114 114 114 T T T 14 280-299 72 72 72 114 114 114 F F F 15 300-319 69 69 69 108 108 108 F F F 16 320-339 72 72 72 115 115 115 F F F 17 340-359 72 72 72 115 115 115 F F F 18 360-379 72 72 72 115 115 115 F F F 19 380-399 71 71 71 112 112 112 F F F 20 400-419 68 68 68 107 107 107 F F F 21 420-439 71 71 71 112 112 112 F F F 22 440-459 67 67 67 106 106 106 F F F 23 460-479 72 72 72 114 114 114 F F F 24 480-499 72 72 72 115 115 115 F F F 25 500-519 72 72 72 115 115 115 F F F 26 520-539 68 68 68 107 107 107 F F F 27 540-559 68 68 68 107 107 107 F F F 28 560-579 72 72 72 115 115 115 F F F 29 580-599 69 69 69 108 108 108 F F F Max gen: 72, Max flush: 115 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 111 111 111 F F F 1 020-039 66 66 66 105 105 105 F F F 2 040-059 73 73 73 116 116 116 F F F 3 060-079 65 65 65 103 103 103 F F F 4 080-099 69 69 69 108 108 108 F F F 5 100-119 73 73 73 116 116 116 F F F 6 120-139 73 73 73 116 116 116 F F F 7 140-159 66 66 66 105 105 105 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 72 72 72 115 115 115 F F F 10 200-219 73 73 73 116 116 116 F F F 11 220-239 72 72 72 115 115 115 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 73 73 73 116 116 116 F F F 14 280-299 72 72 72 114 114 114 F F F 15 300-319 69 69 69 108 108 108 F F F 16 320-339 72 72 72 115 115 115 F F F 17 340-359 72 72 72 115 115 115 F F F 18 360-379 72 72 72 115 115 115 F F F 19 380-399 71 71 71 112 112 112 F F F 20 400-419 68 68 68 107 107 107 F F F 21 420-439 71 71 71 112 112 112 F F F 22 440-459 67 67 67 106 106 106 F F F 23 460-479 72 72 72 114 114 114 F F F 24 480-499 72 72 72 115 115 115 F F F 25 500-519 72 72 72 115 115 115 F F F 26 520-539 68 68 68 107 107 107 F F F 27 540-559 68 68 68 107 107 107 F F F 28 560-579 72 72 72 115 115 115 F F F 29 580-599 69 69 69 108 108 108 F F F Max gen: 73, Max flush: 116 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.485029075Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.529468684Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.530310296Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.530355855Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.530371589Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.530385892Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484} {"msg":"Crucible 7e75aa9b-93f2-4245-b934-3828c00bcbcb has session id: 7ddc1012-54a5-4ae6-8fb7-7118ccaf176f","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.530984901Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"Upstairs opts: Upstairs UUID: 7e75aa9b-93f2-4245-b934-3828c00bcbcb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.53102864Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"Crucible stats registered with UUID: 7e75aa9b-93f2-4245-b934-3828c00bcbcb","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531046364Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:12.531059417Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531638332Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531665588Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484} {"msg":"7e75aa9b-93f2-4245-b934-3828c00bcbcb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531726921Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531745305Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"0","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531776032Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"0","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531800337Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"1","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531826092Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"1","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531852888Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"2","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531883775Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"2","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531910201Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"io task","client":"2","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531939257Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"io task","client":"0","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.531970374Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"io task","client":"1","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46234)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.53199766Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"io task","client":"2","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55776)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.532027686Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"io task","client":"0","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39789)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.532057842Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"io task","client":"1","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.532779728Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"0","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.532811255Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"1","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.533427908Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"1","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.533468007Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"0","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.534090741Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.534215388Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.534341505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"2","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.534894775Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"2","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"[0]R flush_numbers[0..12]: [111, 105, 116, 103, 108, 116, 116, 105, 94, 115, 116, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535453835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"[0]R generation[0..12]: [70, 66, 73, 65, 69, 73, 73, 66, 60, 72, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535480531Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535510418Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"[1]R flush_numbers[0..12]: [111, 105, 116, 103, 108, 116, 116, 105, 94, 115, 116, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535525231Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"[1]R generation[0..12]: [70, 66, 73, 65, 69, 73, 73, 66, 60, 72, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535538433Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535552096Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"[2]R flush_numbers[0..12]: [111, 105, 116, 103, 108, 116, 116, 105, 94, 115, 116, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.53557011Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"[2]R generation[0..12]: [70, 66, 73, 65, 69, 73, 73, 66, 60, 72, 73, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535584223Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535598286Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535610809Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535623612Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535644536Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535657149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"":"downstairs","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535670142Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535682015Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"0","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535694847Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"1","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.53570737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"client":"2","":"downstairs","session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535720233Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535732105Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"7e75aa9b-93f2-4245-b934-3828c00bcbcb is now active with session: 7ddc1012-54a5-4ae6-8fb7-7118ccaf176f","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535744448Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535755911Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484,"session_id":"7ddc1012-54a5-4ae6-8fb7-7118ccaf176f"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535767593Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484} {"msg":"Activated sub_volume 7e75aa9b-93f2-4245-b934-3828c00bcbcb","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:12.535780646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 44 len 32768 data: 11 7 8 8 10 10 10 8 02/30 Read block 90 len 8192 03/30 Flush 04/30 Write block 201 len 36864 data: 8 6 6 6 7 6 5 6 7 05/30 Read block 139 len 12288 06/30 Flush 07/30 Write block 265 len 12288 data: 13 11 10 08/30 Write block 593 len 28672 data: 4 3 2 3 3 3 2 09/30 Read block 98 len 20480 10/30 Read block 582 len 28672 11/30 Write block 510 len 8192 data: 8 5 12/30 Read block 588 len 8192 13/30 Write block 35 len 4096 data: 6 14/30 Read block 181 len 28672 15/30 Write block 379 len 32768 data: 7 6 7 5 6 6 6 6 16/30 Write block 115 len 20480 data: 7 9 8 10 8 17/30 Read block 256 len 28672 18/30 Read block 329 len 16384 19/30 Read block 276 len 28672 20/30 Write block 446 len 12288 data: 10 7 5 21/30 Write block 342 len 40960 data: 5 2 3 3 2 2 4 4 5 5 22/30 Read block 39 len 4096 23/30 Write block 524 len 8192 data: 8 8 24/30 Read block 398 len 28672 25/30 Write block 389 len 8192 data: 6 7 26/30 Write block 377 len 4096 data: 4 27/30 Read block 98 len 16384 28/30 Read block 181 len 32768 29/30 Read block 114 len 12288 30/30 Write block 509 len 12288 data: 8 9 6 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 3 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:12.555454111Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1484} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 111 111 111 F F F 1 020-039 66 66 66 105 105 105 T T T 2 040-059 74 74 74 117 117 117 F F F 3 060-079 65 65 65 103 103 103 F F F 4 080-099 69 69 69 108 108 108 F F F 5 100-119 73 73 73 116 116 116 T T T 6 120-139 73 73 73 116 116 116 F F F 7 140-159 66 66 66 105 105 105 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 72 72 72 115 115 115 F F F 10 200-219 74 74 74 118 118 118 F F F 11 220-239 72 72 72 115 115 115 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 73 73 73 116 116 116 T T T 14 280-299 72 72 72 114 114 114 F F F 15 300-319 69 69 69 108 108 108 F F F 16 320-339 72 72 72 115 115 115 F F F 17 340-359 72 72 72 115 115 115 T T T 18 360-379 72 72 72 115 115 115 T T T 19 380-399 71 71 71 112 112 112 T T T 20 400-419 68 68 68 107 107 107 F F F 21 420-439 71 71 71 112 112 112 F F F 22 440-459 67 67 67 106 106 106 T T T 23 460-479 72 72 72 114 114 114 F F F 24 480-499 72 72 72 115 115 115 F F F 25 500-519 72 72 72 115 115 115 T T T 26 520-539 68 68 68 107 107 107 T T T 27 540-559 68 68 68 107 107 107 F F F 28 560-579 72 72 72 115 115 115 F F F 29 580-599 69 69 69 108 108 108 T T T Max gen: 74, Max flush: 118 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 111 111 111 F F F 1 020-039 75 75 75 119 119 119 F F F 2 040-059 74 74 74 117 117 117 F F F 3 060-079 65 65 65 103 103 103 F F F 4 080-099 69 69 69 108 108 108 F F F 5 100-119 75 75 75 119 119 119 F F F 6 120-139 73 73 73 116 116 116 F F F 7 140-159 66 66 66 105 105 105 F F F 8 160-179 60 60 60 94 94 94 F F F 9 180-199 72 72 72 115 115 115 F F F 10 200-219 74 74 74 118 118 118 F F F 11 220-239 72 72 72 115 115 115 F F F 12 240-259 62 62 62 96 96 96 F F F 13 260-279 75 75 75 119 119 119 F F F 14 280-299 72 72 72 114 114 114 F F F 15 300-319 69 69 69 108 108 108 F F F 16 320-339 72 72 72 115 115 115 F F F 17 340-359 75 75 75 119 119 119 F F F 18 360-379 75 75 75 119 119 119 F F F 19 380-399 75 75 75 119 119 119 F F F 20 400-419 68 68 68 107 107 107 F F F 21 420-439 71 71 71 112 112 112 F F F 22 440-459 75 75 75 119 119 119 F F F 23 460-479 72 72 72 114 114 114 F F F 24 480-499 72 72 72 115 115 115 F F F 25 500-519 75 75 75 119 119 119 F F F 26 520-539 75 75 75 119 119 119 F F F 27 540-559 68 68 68 107 107 107 F F F 28 560-579 72 72 72 115 115 115 F F F 29 580-599 75 75 75 119 119 119 F F F Max gen: 75, Max flush: 119 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.84640928Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:16.888564536Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.888981446Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.919904298Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.920506828Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.920548046Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.92056444Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.920582194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492} {"msg":"Crucible 58cf22c3-385a-466a-9994-167b4aa59232 has session id: 8db34e72-6078-4d02-acf8-85f5540ddfd9","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.92125931Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Upstairs opts: Upstairs UUID: 58cf22c3-385a-466a-9994-167b4aa59232, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.92130666Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Crucible stats registered with UUID: 58cf22c3-385a-466a-9994-167b4aa59232","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.921324834Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:16.921338087Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.921751026Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.921776622Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492} {"msg":"58cf22c3-385a-466a-9994-167b4aa59232 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.921810289Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.921824582Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"0","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.921852558Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"0","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.921871342Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"1","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.921897028Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"1","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.921921843Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"2","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.921948429Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"2","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.921971774Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"io task","client":"2","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.921996339Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"io task","client":"0","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"ds_connection connected from Ok(127.0.0.1:42606)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.922020334Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"io task","client":"2","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"ds_connection connected from Ok(127.0.0.1:53244)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.922042579Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"io task","client":"0","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.922067685Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"io task","client":"1","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37096)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.922217897Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"io task","client":"1","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.922656281Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"1","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.922802953Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"2","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.922982682Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"0","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.923220143Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"1","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.923410244Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"2","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.923554425Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"0","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.923827404Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.92394465Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"[0]R flush_numbers[0..12]: [111, 119, 117, 103, 108, 119, 116, 105, 94, 115, 118, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.92422418Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"[0]R generation[0..12]: [70, 75, 74, 65, 69, 75, 73, 66, 60, 72, 74, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924246705Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924261878Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"[1]R flush_numbers[0..12]: [111, 119, 117, 103, 108, 119, 116, 105, 94, 115, 118, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924275581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"[1]R generation[0..12]: [70, 75, 74, 65, 69, 75, 73, 66, 60, 72, 74, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924288954Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924302967Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"[2]R flush_numbers[0..12]: [111, 119, 117, 103, 108, 119, 116, 105, 94, 115, 118, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.92431707Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"[2]R generation[0..12]: [70, 75, 74, 65, 69, 75, 73, 66, 60, 72, 74, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924330523Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924344666Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924358339Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924374792Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924388205Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924401688Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924420772Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924432435Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"0","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924446258Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"1","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.92445887Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"client":"2","":"downstairs","session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924471533Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924483346Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"58cf22c3-385a-466a-9994-167b4aa59232 is now active with session: 8db34e72-6078-4d02-acf8-85f5540ddfd9","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924495608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924507071Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492,"session_id":"8db34e72-6078-4d02-acf8-85f5540ddfd9"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924518813Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492} {"msg":"Activated sub_volume 58cf22c3-385a-466a-9994-167b4aa59232","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:16.924529546Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 270 len 8192 data: 7 7 02/30 Write block 451 len 12288 data: 7 8 9 03/30 Write block 278 len 36864 data: 5 3 5 6 6 7 8 9 8 04/30 Write block 210 len 24576 data: 5 5 4 3 3 3 05/30 Read block 245 len 8192 06/30 Read block 62 len 40960 07/30 Flush 08/30 Write block 380 len 28672 data: 7 8 6 7 7 7 7 09/30 Write block 320 len 32768 data: 7 6 6 6 7 8 7 7 10/30 Write block 428 len 36864 data: 7 6 7 6 6 7 5 5 5 11/30 Write block 440 len 16384 data: 9 8 10 10 12/30 Write block 257 len 28672 data: 6 7 7 7 6 6 9 13/30 Flush 14/30 Write block 359 len 8192 data: 7 9 15/30 Read block 427 len 40960 16/30 Read block 119 len 4096 17/30 Write block 546 len 36864 data: 6 6 6 6 7 7 8 8 8 18/30 Write block 517 len 8192 data: 10 10 19/30 Read block 51 len 24576 20/30 Write block 54 len 24576 data: 10 9 8 8 8 7 21/30 Write block 7 len 4096 data: 6 22/30 Read block 225 len 36864 23/30 Flush 24/30 Flush 25/30 Write block 163 len 40960 data: 8 11 16 13 8 7 8 6 5 6 26/30 Write block 109 len 36864 data: 3 3 3 3 5 6 8 10 9 27/30 Read block 292 len 8192 28/30 Read block 394 len 40960 29/30 Write block 537 len 40960 data: 4 3 5 5 4 5 5 5 5 7 30/30 Write block 521 len 40960 data: 6 7 8 9 9 10 10 8 7 4 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:16.943336894Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1492} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 122 122 122 F F F 1 020-039 75 75 75 119 119 119 F F F 2 040-059 76 76 76 122 122 122 F F F 3 060-079 65 65 65 103 103 103 F F F 4 080-099 69 69 69 108 108 108 F F F 5 100-119 75 75 75 119 119 119 T T T 6 120-139 73 73 73 116 116 116 F F F 7 140-159 66 66 66 105 105 105 F F F 8 160-179 60 60 60 94 94 94 T T T 9 180-199 72 72 72 115 115 115 F F F 10 200-219 76 76 76 120 120 120 F F F 11 220-239 72 72 72 115 115 115 F F F 12 240-259 76 76 76 121 121 121 F F F 13 260-279 76 76 76 121 121 121 F F F 14 280-299 76 76 76 120 120 120 F F F 15 300-319 69 69 69 108 108 108 F F F 16 320-339 76 76 76 121 121 121 F F F 17 340-359 76 76 76 122 122 122 F F F 18 360-379 76 76 76 122 122 122 F F F 19 380-399 76 76 76 121 121 121 F F F 20 400-419 68 68 68 107 107 107 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 76 76 76 121 121 121 F F F 23 460-479 72 72 72 114 114 114 F F F 24 480-499 72 72 72 115 115 115 F F F 25 500-519 76 76 76 122 122 122 F F F 26 520-539 75 75 75 119 119 119 T T T 27 540-559 76 76 76 122 122 122 T T T 28 560-579 72 72 72 115 115 115 F F F 29 580-599 75 75 75 119 119 119 F F F Max gen: 76, Max flush: 122 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 122 122 122 F F F 1 020-039 75 75 75 119 119 119 F F F 2 040-059 76 76 76 122 122 122 F F F 3 060-079 65 65 65 103 103 103 F F F 4 080-099 69 69 69 108 108 108 F F F 5 100-119 77 77 77 123 123 123 F F F 6 120-139 73 73 73 116 116 116 F F F 7 140-159 66 66 66 105 105 105 F F F 8 160-179 77 77 77 123 123 123 F F F 9 180-199 72 72 72 115 115 115 F F F 10 200-219 76 76 76 120 120 120 F F F 11 220-239 72 72 72 115 115 115 F F F 12 240-259 76 76 76 121 121 121 F F F 13 260-279 76 76 76 121 121 121 F F F 14 280-299 76 76 76 120 120 120 F F F 15 300-319 69 69 69 108 108 108 F F F 16 320-339 76 76 76 121 121 121 F F F 17 340-359 76 76 76 122 122 122 F F F 18 360-379 76 76 76 122 122 122 F F F 19 380-399 76 76 76 121 121 121 F F F 20 400-419 68 68 68 107 107 107 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 76 76 76 121 121 121 F F F 23 460-479 72 72 72 114 114 114 F F F 24 480-499 72 72 72 115 115 115 F F F 25 500-519 76 76 76 122 122 122 F F F 26 520-539 77 77 77 123 123 123 F F F 27 540-559 77 77 77 123 123 123 F F F 28 560-579 72 72 72 115 115 115 F F F 29 580-599 75 75 75 119 119 119 F F F Max gen: 77, Max flush: 123 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.208244433Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.252521737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.253317099Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.25337045Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.253389104Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.253407908Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500} {"msg":"Crucible b26177a5-9b21-4326-b0e4-7c79b27319fb has session id: fc123ae4-722e-4b5f-98a6-c93d748a1772","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.253949255Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Upstairs opts: Upstairs UUID: b26177a5-9b21-4326-b0e4-7c79b27319fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.253995445Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Crucible stats registered with UUID: b26177a5-9b21-4326-b0e4-7c79b27319fb","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254025421Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:21.254040884Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254410114Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.25443432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500} {"msg":"b26177a5-9b21-4326-b0e4-7c79b27319fb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254495633Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254512796Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"0","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254539432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"0","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254553385Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"1","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254567328Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"1","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254580581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"2","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254604476Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"2","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254630072Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"io task","client":"2","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254651916Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"io task","client":"0","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254675581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"io task","client":"1","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"ds_connection connected from Ok(127.0.0.1:47205)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.254771302Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"io task","client":"1","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"ds_connection connected from Ok(127.0.0.1:65002)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.25480883Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"io task","client":"0","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:21.254915883Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"io task","client":"2","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:21.254958073Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"io task","client":"2","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:21.254986909Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.255168418Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"io task","client":"2","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.255600431Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"0","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.255637519Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"1","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.256310604Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"0","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.256343972Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"1","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.25694097Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:21.257040342Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.25619552Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"io task","client":"2","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"ds_connection connected from Ok(127.0.0.1:59814)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.256308854Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"io task","client":"2","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.257316322Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"2","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.257834113Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"2","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"[0]R flush_numbers[0..12]: [122, 119, 122, 103, 108, 123, 116, 105, 123, 115, 120, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258408857Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"[0]R generation[0..12]: [76, 75, 76, 65, 69, 77, 73, 66, 77, 72, 76, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258433733Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258449406Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"[1]R flush_numbers[0..12]: [122, 119, 122, 103, 108, 123, 116, 105, 123, 115, 120, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258481823Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"[1]R generation[0..12]: [76, 75, 76, 65, 69, 77, 73, 66, 77, 72, 76, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258496016Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258509639Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"[2]R flush_numbers[0..12]: [122, 119, 122, 103, 108, 123, 116, 105, 123, 115, 120, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258523082Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"[2]R generation[0..12]: [76, 75, 76, 65, 69, 77, 73, 66, 77, 72, 76, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258536335Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258550028Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258562591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258575023Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Next flush: 124","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258587326Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258599788Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"":"downstairs","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258616422Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258628465Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"0","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258641477Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"1","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258655451Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"client":"2","":"downstairs","session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258668283Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:23:31.258679926Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"b26177a5-9b21-4326-b0e4-7c79b27319fb is now active with session: fc123ae4-722e-4b5f-98a6-c93d748a1772","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.25870008Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258714723Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500,"session_id":"fc123ae4-722e-4b5f-98a6-c93d748a1772"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258729387Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500} {"msg":"Activated sub_volume b26177a5-9b21-4326-b0e4-7c79b27319fb","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:31.258742799Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 7 len 4096 data: 7 02/30 Flush 03/30 Write block 205 len 36864 data: 8 7 6 7 8 6 6 5 4 04/30 Read block 230 len 12288 05/30 Read block 69 len 8192 06/30 Read block 544 len 16384 07/30 Read block 81 len 28672 08/30 Read block 189 len 36864 09/30 Flush 10/30 Write block 538 len 28672 data: 4 6 6 5 6 6 6 11/30 Read block 462 len 28672 12/30 Write block 230 len 36864 data: 10 7 6 7 7 8 7 6 6 13/30 Read block 497 len 20480 14/30 Write block 517 len 12288 data: 11 11 8 15/30 Read block 454 len 24576 16/30 Flush 17/30 Write block 297 len 40960 data: 9 10 7 8 8 7 5 5 5 4 18/30 Write block 478 len 28672 data: 5 5 5 5 6 8 8 19/30 Read block 129 len 12288 20/30 Read block 388 len 32768 21/30 Write block 220 len 8192 data: 6 6 22/30 Write block 228 len 32768 data: 9 8 11 8 7 8 8 9 23/30 Read block 13 len 32768 24/30 Read block 384 len 4096 25/30 Read block 37 len 36864 26/30 Write block 27 len 16384 data: 5 7 7 5 27/30 Write block 48 len 8192 data: 11 11 28/30 Write block 153 len 32768 data: 6 9 9 9 7 7 8 8 29/30 Read block 229 len 28672 30/30 Write block 5 len 24576 data: 4 6 8 6 5 6 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 10 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:31.280349692Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1500} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 124 124 124 T T T 1 020-039 75 75 75 119 119 119 T T T 2 040-059 76 76 76 122 122 122 T T T 3 060-079 65 65 65 103 103 103 F F F 4 080-099 69 69 69 108 108 108 F F F 5 100-119 77 77 77 123 123 123 F F F 6 120-139 73 73 73 116 116 116 F F F 7 140-159 66 66 66 105 105 105 T T T 8 160-179 77 77 77 123 123 123 T T T 9 180-199 72 72 72 115 115 115 F F F 10 200-219 78 78 78 125 125 125 F F F 11 220-239 78 78 78 126 126 126 T T T 12 240-259 76 76 76 121 121 121 F F F 13 260-279 76 76 76 121 121 121 F F F 14 280-299 76 76 76 120 120 120 T T T 15 300-319 69 69 69 108 108 108 T T T 16 320-339 76 76 76 121 121 121 F F F 17 340-359 76 76 76 122 122 122 F F F 18 360-379 76 76 76 122 122 122 F F F 19 380-399 76 76 76 121 121 121 F F F 20 400-419 68 68 68 107 107 107 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 76 76 76 121 121 121 F F F 23 460-479 72 72 72 114 114 114 T T T 24 480-499 72 72 72 115 115 115 T T T 25 500-519 78 78 78 126 126 126 F F F 26 520-539 78 78 78 126 126 126 F F F 27 540-559 78 78 78 126 126 126 F F F 28 560-579 72 72 72 115 115 115 F F F 29 580-599 75 75 75 119 119 119 F F F Max gen: 78, Max flush: 126 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 127 127 127 F F F 1 020-039 79 79 79 127 127 127 F F F 2 040-059 79 79 79 127 127 127 F F F 3 060-079 65 65 65 103 103 103 F F F 4 080-099 69 69 69 108 108 108 F F F 5 100-119 77 77 77 123 123 123 F F F 6 120-139 73 73 73 116 116 116 F F F 7 140-159 79 79 79 127 127 127 F F F 8 160-179 79 79 79 127 127 127 F F F 9 180-199 72 72 72 115 115 115 F F F 10 200-219 78 78 78 125 125 125 F F F 11 220-239 79 79 79 127 127 127 F F F 12 240-259 76 76 76 121 121 121 F F F 13 260-279 76 76 76 121 121 121 F F F 14 280-299 79 79 79 127 127 127 F F F 15 300-319 79 79 79 127 127 127 F F F 16 320-339 76 76 76 121 121 121 F F F 17 340-359 76 76 76 122 122 122 F F F 18 360-379 76 76 76 122 122 122 F F F 19 380-399 76 76 76 121 121 121 F F F 20 400-419 68 68 68 107 107 107 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 76 76 76 121 121 121 F F F 23 460-479 79 79 79 127 127 127 F F F 24 480-499 79 79 79 127 127 127 F F F 25 500-519 78 78 78 126 126 126 F F F 26 520-539 78 78 78 126 126 126 F F F 27 540-559 78 78 78 126 126 126 F F F 28 560-579 72 72 72 115 115 115 F F F 29 580-599 75 75 75 119 119 119 F F F Max gen: 79, Max flush: 127 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.548184832Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.591858986Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.592607047Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.592647906Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.592663449Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.592677802Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508} {"msg":"Crucible 8bf8372e-f007-428c-ba78-cca1d1112bcf has session id: 617c86c9-ba0d-4a7b-91d0-4ffff7739b1d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593236083Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Upstairs opts: Upstairs UUID: 8bf8372e-f007-428c-ba78-cca1d1112bcf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593277822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Crucible stats registered with UUID: 8bf8372e-f007-428c-ba78-cca1d1112bcf","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593301077Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:45.593321051Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593748153Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.59378081Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508} {"msg":"8bf8372e-f007-428c-ba78-cca1d1112bcf active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593843344Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593861068Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"0","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593887133Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"0","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593900956Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"1","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593915209Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"1","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593928122Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"2","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593941625Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"2","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593959629Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"io task","client":"2","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.593982824Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"io task","client":"0","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.594005959Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57694)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.594037616Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"io task","client":"2","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60590)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.594066032Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"io task","client":"0","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:45.594220555Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:45.594303663Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:45.594332179Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.59442582Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.594879327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"2","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.594908534Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"0","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.595459643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"0","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.595557554Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"2","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.595998669Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:45.596080466Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.595762799Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62049)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.59590923Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"io task","client":"1","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.596912527Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"1","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.597456164Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"1","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"[0]R flush_numbers[0..12]: [127, 127, 127, 103, 108, 123, 116, 127, 127, 115, 125, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598079409Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"[0]R generation[0..12]: [79, 79, 79, 65, 69, 77, 73, 79, 79, 72, 78, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598109765Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.59812988Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"[1]R flush_numbers[0..12]: [127, 127, 127, 103, 108, 123, 116, 127, 127, 115, 125, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598147233Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"[1]R generation[0..12]: [79, 79, 79, 65, 69, 77, 73, 79, 79, 72, 78, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598163977Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598180881Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"[2]R flush_numbers[0..12]: [127, 127, 127, 103, 108, 123, 116, 127, 127, 115, 125, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598198095Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"[2]R generation[0..12]: [79, 79, 79, 65, 69, 77, 73, 79, 79, 72, 78, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598215278Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598232642Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598248345Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598264569Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598280222Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598296016Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"":"downstairs","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598311949Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598326232Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"Wait for a query_work_queue command to finish before sending IO client":"0","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598370192Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"1","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598390866Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"client":"2","":"downstairs","session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.59840747Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598422393Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"8bf8372e-f007-428c-ba78-cca1d1112bcf is now active with session: 617c86c9-ba0d-4a7b-91d0-4ffff7739b1d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598437956Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598452379Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508,"session_id":"617c86c9-ba0d-4a7b-91d0-4ffff7739b1d"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598467173Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508} {"msg":"Activated sub_volume 8bf8372e-f007-428c-ba78-cca1d1112bcf","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:55.598480515Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 320 len 16384 data: 8 7 7 7 02/30 Write block 541 len 20480 data: 6 7 7 7 6 03/30 Write block 139 len 20480 data: 7 5 4 5 5 04/30 Read block 139 len 12288 05/30 Read block 445 len 8192 06/30 Write block 3 len 20480 data: 4 6 5 7 9 07/30 Read block 143 len 8192 08/30 Read block 63 len 20480 09/30 Read block 366 len 4096 10/30 Flush 11/30 Write block 452 len 24576 data: 9 10 8 6 4 4 12/30 Read block 560 len 12288 13/30 Flush 14/30 Flush 15/30 Flush 16/30 Write block 142 len 36864 data: 6 6 5 5 4 6 7 7 6 17/30 Read block 347 len 4096 18/30 Write block 67 len 24576 data: 8 8 8 10 8 10 19/30 Write block 309 len 4096 data: 4 20/30 Write block 135 len 28672 data: 5 5 4 6 8 6 5 21/30 Write block 382 len 8192 data: 7 8 22/30 Write block 150 len 20480 data: 7 8 7 7 10 23/30 Read block 290 len 40960 24/30 Read block 68 len 20480 25/30 Flush 26/30 Write block 23 len 16384 data: 5 4 2 5 27/30 Read block 527 len 20480 28/30 Write block 44 len 8192 data: 12 8 29/30 Write block 91 len 36864 data: 7 6 6 4 4 3 5 5 5 30/30 Write block 578 len 32768 data: 7 7 5 7 6 7 10 10 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:55.617609903Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1508} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 128 128 128 F F F 1 020-039 79 79 79 127 127 127 T T T 2 040-059 79 79 79 127 127 127 T T T 3 060-079 80 80 80 132 132 132 F F F 4 080-099 69 69 69 108 108 108 T T T 5 100-119 77 77 77 123 123 123 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 80 80 80 132 132 132 F F F 8 160-179 79 79 79 127 127 127 F F F 9 180-199 72 72 72 115 115 115 F F F 10 200-219 78 78 78 125 125 125 F F F 11 220-239 79 79 79 127 127 127 F F F 12 240-259 76 76 76 121 121 121 F F F 13 260-279 76 76 76 121 121 121 F F F 14 280-299 79 79 79 127 127 127 F F F 15 300-319 80 80 80 132 132 132 F F F 16 320-339 80 80 80 128 128 128 F F F 17 340-359 76 76 76 122 122 122 F F F 18 360-379 76 76 76 122 122 122 F F F 19 380-399 80 80 80 132 132 132 F F F 20 400-419 68 68 68 107 107 107 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 80 80 80 129 129 129 F F F 23 460-479 79 79 79 127 127 127 F F F 24 480-499 79 79 79 127 127 127 F F F 25 500-519 78 78 78 126 126 126 F F F 26 520-539 78 78 78 126 126 126 F F F 27 540-559 80 80 80 128 128 128 F F F 28 560-579 72 72 72 115 115 115 T T T 29 580-599 75 75 75 119 119 119 T T T Max gen: 80, Max flush: 132 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 128 128 128 F F F 1 020-039 81 81 81 133 133 133 F F F 2 040-059 81 81 81 133 133 133 F F F 3 060-079 80 80 80 132 132 132 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 77 77 77 123 123 123 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 80 80 80 132 132 132 F F F 8 160-179 79 79 79 127 127 127 F F F 9 180-199 72 72 72 115 115 115 F F F 10 200-219 78 78 78 125 125 125 F F F 11 220-239 79 79 79 127 127 127 F F F 12 240-259 76 76 76 121 121 121 F F F 13 260-279 76 76 76 121 121 121 F F F 14 280-299 79 79 79 127 127 127 F F F 15 300-319 80 80 80 132 132 132 F F F 16 320-339 80 80 80 128 128 128 F F F 17 340-359 76 76 76 122 122 122 F F F 18 360-379 76 76 76 122 122 122 F F F 19 380-399 80 80 80 132 132 132 F F F 20 400-419 68 68 68 107 107 107 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 80 80 80 129 129 129 F F F 23 460-479 79 79 79 127 127 127 F F F 24 480-499 79 79 79 127 127 127 F F F 25 500-519 78 78 78 126 126 126 F F F 26 520-539 78 78 78 126 126 126 F F F 27 540-559 80 80 80 128 128 128 F F F 28 560-579 81 81 81 133 133 133 F F F 29 580-599 81 81 81 133 133 133 F F F Max gen: 81, Max flush: 133 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.858005953Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.902617329Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.903292835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.903336804Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.903351967Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.903366441Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516} {"msg":"Crucible 23cb4e13-0d3e-4d59-a21a-ad8c1167f743 has session id: 011f0b2f-fde1-4884-b09a-9d595c8075e3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.90401452Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"Upstairs opts: Upstairs UUID: 23cb4e13-0d3e-4d59-a21a-ad8c1167f743, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.90406035Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"Crucible stats registered with UUID: 23cb4e13-0d3e-4d59-a21a-ad8c1167f743","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904083795Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:59.90410453Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904604838Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904628393Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516} {"msg":"23cb4e13-0d3e-4d59-a21a-ad8c1167f743 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904677664Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904704209Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"0","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904734866Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"0","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.90475513Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"1","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904779365Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"1","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904804031Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"2","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904822205Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"2","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904841509Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"io task","client":"2","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904866274Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"io task","client":"0","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904889919Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"io task","client":"1","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:60275)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904921716Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"io task","client":"1","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43006)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904951243Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"io task","client":"2","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:60944)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.904977098Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"io task","client":"0","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.905606744Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"1","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.905767689Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"0","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.905807497Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"2","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.906168665Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"1","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.906212765Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"0","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.906493215Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"2","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.906787239Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.906902464Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"[0]R flush_numbers[0..12]: [128, 133, 133, 132, 133, 123, 132, 132, 127, 115, 125, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.90716657Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"[0]R generation[0..12]: [80, 81, 81, 80, 81, 77, 80, 80, 79, 72, 78, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907190126Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907205009Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"[1]R flush_numbers[0..12]: [128, 133, 133, 132, 133, 123, 132, 132, 127, 115, 125, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907219432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"[1]R generation[0..12]: [80, 81, 81, 80, 81, 77, 80, 80, 79, 72, 78, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907233225Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907247098Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"[2]R flush_numbers[0..12]: [128, 133, 133, 132, 133, 123, 132, 132, 127, 115, 125, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907260981Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"[2]R generation[0..12]: [80, 81, 81, 80, 81, 77, 80, 80, 79, 72, 78, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907274414Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907288057Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907300589Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907317833Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907336967Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.90734983Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"":"downstairs","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907362603Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907374235Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"0","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907387288Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"1","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907400331Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"client":"2","":"downstairs","session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907413194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907425026Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"23cb4e13-0d3e-4d59-a21a-ad8c1167f743 is now active with session: 011f0b2f-fde1-4884-b09a-9d595c8075e3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907437329Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907448841Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516,"session_id":"011f0b2f-fde1-4884-b09a-9d595c8075e3"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907460644Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516} {"msg":"Activated sub_volume 23cb4e13-0d3e-4d59-a21a-ad8c1167f743","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:23:59.907471486Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 307 len 28672 data: 4 5 5 3 5 5 6 02/30 Read block 51 len 36864 03/30 Read block 496 len 16384 04/30 Read block 438 len 36864 05/30 Read block 159 len 32768 06/30 Write block 457 len 36864 data: 5 4 4 5 5 6 8 8 9 07/30 Write block 105 len 20480 data: 4 4 5 4 4 08/30 Read block 148 len 16384 09/30 Read block 56 len 12288 10/30 Write block 568 len 24576 data: 9 9 8 9 8 8 11/30 Write block 49 len 4096 data: 12 12/30 Read block 438 len 28672 13/30 Write block 248 len 8192 data: 4 6 14/30 Write block 302 len 36864 data: 8 6 6 6 5 5 6 6 4 15/30 Write block 542 len 32768 data: 8 8 8 7 8 7 7 7 16/30 Flush 17/30 Write block 398 len 16384 data: 6 7 7 6 18/30 Write block 31 len 4096 data: 5 19/30 Read block 269 len 24576 20/30 Write block 58 len 32768 data: 9 8 7 6 7 7 8 7 21/30 Write block 8 len 28672 data: 7 6 7 6 7 4 3 22/30 Read block 256 len 4096 23/30 Read block 233 len 16384 24/30 Read block 185 len 28672 25/30 Read block 133 len 40960 26/30 Read block 146 len 20480 27/30 Write block 533 len 40960 data: 4 4 4 4 5 5 7 7 7 9 28/30 Write block 434 len 4096 data: 6 29/30 Write block 12 len 36864 data: 8 5 4 3 4 3 4 4 6 30/30 Write block 389 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 4 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 1 Done Done Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 3 7 Done 12 12 11 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:23:59.927959417Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1516} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 128 128 128 T T T 1 020-039 81 81 81 133 133 133 T T T 2 040-059 82 82 82 134 134 134 T T T 3 060-079 80 80 80 132 132 132 T T T 4 080-099 81 81 81 133 133 133 F F F 5 100-119 82 82 82 134 134 134 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 80 80 80 132 132 132 F F F 8 160-179 79 79 79 127 127 127 F F F 9 180-199 72 72 72 115 115 115 F F F 10 200-219 78 78 78 125 125 125 F F F 11 220-239 79 79 79 127 127 127 F F F 12 240-259 82 82 82 134 134 134 F F F 13 260-279 76 76 76 121 121 121 F F F 14 280-299 79 79 79 127 127 127 F F F 15 300-319 82 82 82 134 134 134 F F F 16 320-339 80 80 80 128 128 128 F F F 17 340-359 76 76 76 122 122 122 F F F 18 360-379 76 76 76 122 122 122 F F F 19 380-399 80 80 80 132 132 132 T T T 20 400-419 68 68 68 107 107 107 T T T 21 420-439 76 76 76 121 121 121 T T T 22 440-459 82 82 82 134 134 134 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 79 79 79 127 127 127 F F F 25 500-519 78 78 78 126 126 126 F F F 26 520-539 78 78 78 126 126 126 T T T 27 540-559 82 82 82 134 134 134 T T T 28 560-579 82 82 82 134 134 134 F F F 29 580-599 81 81 81 133 133 133 F F F Max gen: 82, Max flush: 134 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 135 135 135 F F F 1 020-039 83 83 83 135 135 135 F F F 2 040-059 83 83 83 135 135 135 F F F 3 060-079 83 83 83 135 135 135 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 82 82 82 134 134 134 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 80 80 80 132 132 132 F F F 8 160-179 79 79 79 127 127 127 F F F 9 180-199 72 72 72 115 115 115 F F F 10 200-219 78 78 78 125 125 125 F F F 11 220-239 79 79 79 127 127 127 F F F 12 240-259 82 82 82 134 134 134 F F F 13 260-279 76 76 76 121 121 121 F F F 14 280-299 79 79 79 127 127 127 F F F 15 300-319 82 82 82 134 134 134 F F F 16 320-339 80 80 80 128 128 128 F F F 17 340-359 76 76 76 122 122 122 F F F 18 360-379 76 76 76 122 122 122 F F F 19 380-399 83 83 83 135 135 135 F F F 20 400-419 83 83 83 135 135 135 F F F 21 420-439 83 83 83 135 135 135 F F F 22 440-459 82 82 82 134 134 134 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 79 79 79 127 127 127 F F F 25 500-519 78 78 78 126 126 126 F F F 26 520-539 83 83 83 135 135 135 F F F 27 540-559 83 83 83 135 135 135 F F F 28 560-579 82 82 82 134 134 134 F F F 29 580-599 81 81 81 133 133 133 F F F Max gen: 83, Max flush: 135 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.18901931Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.23355906Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.234270734Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.234312273Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.234326696Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.234340959Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524} {"msg":"Crucible 8307b3e3-35d9-4039-b170-66c9d058ca16 has session id: 0d3fb404-36db-456c-b460-4dfc3491b704","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.234921804Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Upstairs opts: Upstairs UUID: 8307b3e3-35d9-4039-b170-66c9d058ca16, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.234969334Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Crucible stats registered with UUID: 8307b3e3-35d9-4039-b170-66c9d058ca16","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.234992429Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:14.235012284Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235403868Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235435835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524} {"msg":"8307b3e3-35d9-4039-b170-66c9d058ca16 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235487246Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235514272Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"0","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235544928Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"0","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235559642Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"1","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235574185Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"1","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235684979Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"2","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235719416Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"2","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235747482Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"io task","client":"2","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235773798Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"io task","client":"0","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235797793Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"io task","client":"1","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"ds_connection connected from Ok(127.0.0.1:39375)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.23582925Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"io task","client":"2","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"ds_connection connected from Ok(127.0.0.1:33714)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235856626Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"io task","client":"0","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"ds_connection connected from Ok(127.0.0.1:51243)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.235883822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"io task","client":"1","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.236455015Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"0","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.236484301Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"1","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.237148034Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"1","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.237176791Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"0","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.237815238Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.237941616Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.238026934Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"2","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.238512699Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"2","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"[0]R flush_numbers[0..12]: [135, 135, 135, 135, 133, 134, 132, 132, 127, 115, 125, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.23902519Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"[0]R generation[0..12]: [83, 83, 83, 83, 81, 82, 80, 80, 79, 72, 78, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239045244Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239059537Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"[1]R flush_numbers[0..12]: [135, 135, 135, 135, 133, 134, 132, 132, 127, 115, 125, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239077071Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"[1]R generation[0..12]: [83, 83, 83, 83, 81, 82, 80, 80, 79, 72, 78, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239091024Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239105817Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"[2]R flush_numbers[0..12]: [135, 135, 135, 135, 133, 134, 132, 132, 127, 115, 125, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.23911909Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"[2]R generation[0..12]: [83, 83, 83, 83, 81, 82, 80, 80, 79, 72, 78, 79]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239132873Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239146486Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239158808Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239171631Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239183934Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239196507Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"":"downstairs","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239209329Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239221142Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"0","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239234124Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"1","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239246737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"client":"2","":"downstairs","session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:24:14.23925965Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239280635Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"8307b3e3-35d9-4039-b170-66c9d058ca16 is now active with session: 0d3fb404-36db-456c-b460-4dfc3491b704","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239294277Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.23930603Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524,"session_id":"0d3fb404-36db-456c-b460-4dfc3491b704"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239317903Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524} {"msg":"Activated sub_volume 8307b3e3-35d9-4039-b170-66c9d058ca16","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:14.239328765Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 420 len 4096 data: 4 02/30 Write block 524 len 40960 data: 10 10 11 11 9 8 5 4 3 5 03/30 Read block 240 len 24576 04/30 Read block 323 len 8192 05/30 Read block 503 len 4096 06/30 Read block 435 len 28672 07/30 Write block 339 len 24576 data: 8 7 5 6 3 4 08/30 Read block 432 len 32768 09/30 Read block 486 len 24576 10/30 Read block 124 len 40960 11/30 Write block 423 len 40960 data: 5 6 5 6 7 8 7 8 7 7 12/30 Write block 485 len 32768 data: 9 9 9 10 13 13 13 13 13/30 Read block 11 len 24576 14/30 Write block 539 len 32768 data: 8 8 8 10 9 9 8 9 15/30 Write block 375 len 32768 data: 7 5 5 6 8 8 9 8 16/30 Write block 232 len 40960 data: 8 9 9 10 8 7 7 6 5 4 17/30 Read block 32 len 8192 18/30 Flush 19/30 Write block 210 len 12288 data: 7 7 6 20/30 Write block 151 len 8192 data: 9 8 21/30 Read block 513 len 16384 22/30 Write block 197 len 36864 data: 6 5 5 8 9 7 7 7 9 23/30 Write block 406 len 36864 data: 7 10 8 6 5 5 5 4 5 24/30 Write block 0 len 36864 data: 3 4 5 5 7 6 8 10 8 25/30 Write block 573 len 4096 data: 9 26/30 Read block 293 len 36864 27/30 Read block 64 len 16384 28/30 Write block 544 len 16384 data: 10 9 10 8 29/30 Write block 446 len 8192 data: 11 8 30/30 Read block 18 len 12288 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 3 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 4 Done Done Sent false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 3 3 Done 12 12 9 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:14.260174403Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1524} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 135 135 135 T T T 1 020-039 83 83 83 135 135 135 F F F 2 040-059 83 83 83 135 135 135 F F F 3 060-079 83 83 83 135 135 135 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 82 82 82 134 134 134 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 80 80 80 132 132 132 T T T 8 160-179 79 79 79 127 127 127 F F F 9 180-199 72 72 72 115 115 115 T T T 10 200-219 78 78 78 125 125 125 T T T 11 220-239 84 84 84 136 136 136 F F F 12 240-259 84 84 84 136 136 136 F F F 13 260-279 76 76 76 121 121 121 F F F 14 280-299 79 79 79 127 127 127 F F F 15 300-319 82 82 82 134 134 134 F F F 16 320-339 84 84 84 136 136 136 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 84 84 84 136 136 136 F F F 19 380-399 84 84 84 136 136 136 F F F 20 400-419 83 83 83 135 135 135 T T T 21 420-439 84 84 84 136 136 136 F F F 22 440-459 82 82 82 134 134 134 T T T 23 460-479 82 82 82 134 134 134 F F F 24 480-499 84 84 84 136 136 136 F F F 25 500-519 78 78 78 126 126 126 F F F 26 520-539 84 84 84 136 136 136 F F F 27 540-559 84 84 84 136 136 136 T T T 28 560-579 82 82 82 134 134 134 T T T 29 580-599 81 81 81 133 133 133 F F F Max gen: 84, Max flush: 136 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 85 85 85 137 137 137 F F F 1 020-039 83 83 83 135 135 135 F F F 2 040-059 83 83 83 135 135 135 F F F 3 060-079 83 83 83 135 135 135 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 82 82 82 134 134 134 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 85 85 85 137 137 137 F F F 8 160-179 79 79 79 127 127 127 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 85 85 85 137 137 137 F F F 11 220-239 84 84 84 136 136 136 F F F 12 240-259 84 84 84 136 136 136 F F F 13 260-279 76 76 76 121 121 121 F F F 14 280-299 79 79 79 127 127 127 F F F 15 300-319 82 82 82 134 134 134 F F F 16 320-339 84 84 84 136 136 136 F F F 17 340-359 84 84 84 136 136 136 F F F 18 360-379 84 84 84 136 136 136 F F F 19 380-399 84 84 84 136 136 136 F F F 20 400-419 85 85 85 137 137 137 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 85 85 85 137 137 137 F F F 23 460-479 82 82 82 134 134 134 F F F 24 480-499 84 84 84 136 136 136 F F F 25 500-519 78 78 78 126 126 126 F F F 26 520-539 84 84 84 136 136 136 F F F 27 540-559 85 85 85 137 137 137 F F F 28 560-579 85 85 85 137 137 137 F F F 29 580-599 81 81 81 133 133 133 F F F Max gen: 85, Max flush: 137 Begin loop 43 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.517640038Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.561357541Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.562105833Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.562147492Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.562163485Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.562177908Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532} {"msg":"Crucible d99ccce4-7979-44ab-982d-249aa7ee249e has session id: 2711c392-d933-48a7-9c68-4094e422b401","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.562851384Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"Upstairs opts: Upstairs UUID: d99ccce4-7979-44ab-982d-249aa7ee249e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.562897203Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"Crucible stats registered with UUID: d99ccce4-7979-44ab-982d-249aa7ee249e","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.562921679Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:28.562936602Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.56353184Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563557156Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532} {"msg":"d99ccce4-7979-44ab-982d-249aa7ee249e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563638714Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563674001Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"0","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563703968Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"0","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563726922Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"1","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563753528Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"1","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563778744Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"2","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563845118Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"2","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563883826Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"io task","client":"2","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563909842Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"io task","client":"0","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563937938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"io task","client":"1","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"ds_connection connected from Ok(127.0.0.1:42438)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563963814Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"io task","client":"2","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"ds_connection connected from Ok(127.0.0.1:35810)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.563988819Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"io task","client":"0","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"ds_connection connected from Ok(127.0.0.1:43198)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.564157855Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"io task","client":"1","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.5646879Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"0","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.564842543Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"1","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.565307754Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"0","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.565452705Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"1","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.565884338Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.566018207Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.566043993Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"2","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.566627518Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"2","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"[0]R flush_numbers[0..12]: [137, 135, 135, 135, 133, 134, 132, 137, 127, 137, 137, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567164054Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"[0]R generation[0..12]: [85, 83, 83, 83, 81, 82, 80, 85, 79, 85, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567184639Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567199402Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"[1]R flush_numbers[0..12]: [137, 135, 135, 135, 133, 134, 132, 137, 127, 137, 137, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567212925Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"[1]R generation[0..12]: [85, 83, 83, 83, 81, 82, 80, 85, 79, 85, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567235239Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567260745Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"[2]R flush_numbers[0..12]: [137, 135, 135, 135, 133, 134, 132, 137, 127, 137, 137, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567287011Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"[2]R generation[0..12]: [85, 83, 83, 83, 81, 82, 80, 85, 79, 85, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567303464Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567324949Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567348594Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.56737842Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567396684Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567409707Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"":"downstairs","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567429961Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"session_id":"2711c392-d933-48a7-9c68-4094e422b401"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567457027Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"0","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567482473Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"1","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567502167Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"client":"2","":"downstairs","session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.56751545Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567527413Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"d99ccce4-7979-44ab-982d-249aa7ee249e is now active with session: 2711c392-d933-48a7-9c68-4094e422b401","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567548757Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567571142Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532,"session_id":"2711c392-d933-48a7-9c68-4094e422b401"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.567594527Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532} {"msg":"Activated sub_volume d99ccce4-7979-44ab-982d-249aa7ee249e","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:28.56760873Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 335 len 36864 data: 12 10 8 8 9 8 6 7 4 03/30 Read block 485 len 24576 04/30 Write block 473 len 20480 data: 9 9 8 6 5 05/30 Write block 201 len 4096 data: 10 06/30 Read block 313 len 16384 07/30 Read block 247 len 8192 08/30 Write block 303 len 28672 data: 7 7 7 6 6 7 7 09/30 Read block 386 len 20480 10/30 Read block 522 len 28672 11/30 Read block 23 len 24576 12/30 Write block 301 len 16384 data: 9 9 8 8 13/30 Flush 14/30 Write block 176 len 12288 data: 7 9 10 15/30 Read block 411 len 40960 16/30 Write block 255 len 28672 data: 6 6 7 8 8 8 7 17/30 Read block 30 len 20480 18/30 Read block 248 len 36864 19/30 Write block 46 len 24576 data: 9 9 12 13 11 9 20/30 Write block 393 len 28672 data: 7 6 6 7 7 7 8 21/30 Write block 100 len 36864 data: 3 4 4 4 5 5 5 6 5 22/30 Write block 227 len 16384 data: 9 10 9 12 23/30 Read block 564 len 36864 24/30 Write block 157 len 32768 data: 8 8 9 9 8 6 9 12 25/30 Read block 307 len 36864 26/30 Flush 27/30 Write block 453 len 28672 data: 11 9 7 5 6 5 5 28/30 Read block 439 len 4096 29/30 Read block 346 len 4096 30/30 Write block 457 len 32768 data: 7 6 6 6 6 7 9 9 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:28.588631316Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1532} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 85 85 85 137 137 137 F F F 1 020-039 83 83 83 135 135 135 F F F 2 040-059 86 86 86 140 140 140 F F F 3 060-079 83 83 83 135 135 135 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 86 86 86 140 140 140 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 86 86 86 140 140 140 F F F 8 160-179 86 86 86 140 140 140 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 86 86 86 139 139 139 F F F 11 220-239 86 86 86 140 140 140 F F F 12 240-259 86 86 86 140 140 140 F F F 13 260-279 86 86 86 140 140 140 F F F 14 280-299 79 79 79 127 127 127 F F F 15 300-319 86 86 86 139 139 139 F F F 16 320-339 86 86 86 139 139 139 F F F 17 340-359 86 86 86 139 139 139 F F F 18 360-379 84 84 84 136 136 136 F F F 19 380-399 86 86 86 140 140 140 F F F 20 400-419 85 85 85 137 137 137 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 85 85 85 137 137 137 T T T 23 460-479 86 86 86 139 139 139 T T T 24 480-499 84 84 84 136 136 136 F F F 25 500-519 78 78 78 126 126 126 F F F 26 520-539 84 84 84 136 136 136 F F F 27 540-559 85 85 85 137 137 137 F F F 28 560-579 85 85 85 137 137 137 F F F 29 580-599 81 81 81 133 133 133 F F F Max gen: 86, Max flush: 140 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 85 85 85 137 137 137 F F F 1 020-039 83 83 83 135 135 135 F F F 2 040-059 86 86 86 140 140 140 F F F 3 060-079 83 83 83 135 135 135 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 86 86 86 140 140 140 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 86 86 86 140 140 140 F F F 8 160-179 86 86 86 140 140 140 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 86 86 86 139 139 139 F F F 11 220-239 86 86 86 140 140 140 F F F 12 240-259 86 86 86 140 140 140 F F F 13 260-279 86 86 86 140 140 140 F F F 14 280-299 79 79 79 127 127 127 F F F 15 300-319 86 86 86 139 139 139 F F F 16 320-339 86 86 86 139 139 139 F F F 17 340-359 86 86 86 139 139 139 F F F 18 360-379 84 84 84 136 136 136 F F F 19 380-399 86 86 86 140 140 140 F F F 20 400-419 85 85 85 137 137 137 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 87 87 87 141 141 141 F F F 23 460-479 87 87 87 141 141 141 F F F 24 480-499 84 84 84 136 136 136 F F F 25 500-519 78 78 78 126 126 126 F F F 26 520-539 84 84 84 136 136 136 F F F 27 540-559 85 85 85 137 137 137 F F F 28 560-579 85 85 85 137 137 137 F F F 29 580-599 81 81 81 133 133 133 F F F Max gen: 87, Max flush: 141 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.817775548Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.861495471Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.86232413Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.86236871Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.862383233Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.862397496Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540} {"msg":"Crucible 13e28404-eb9f-4553-8f96-65b8d3daf84d has session id: 58f5ef2c-a958-4c2c-9fb7-74b343975f04","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863012889Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Upstairs opts: Upstairs UUID: 13e28404-eb9f-4553-8f96-65b8d3daf84d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863055958Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Crucible stats registered with UUID: 13e28404-eb9f-4553-8f96-65b8d3daf84d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863072752Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:32.863089966Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863535372Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863559057Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540} {"msg":"13e28404-eb9f-4553-8f96-65b8d3daf84d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863629592Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863646586Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"0","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863674622Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"0","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863688415Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"1","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863702488Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"1","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863723522Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"2","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863751689Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"2","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863778354Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"io task","client":"2","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.86380527Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"io task","client":"0","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863829195Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"io task","client":"1","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"ds_connection connected from Ok(127.0.0.1:52500)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.863855281Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"io task","client":"0","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:32.863886467Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"io task","client":"2","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:32.863924426Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"io task","client":"2","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:32.864078449Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"ds_connection connected from Ok(127.0.0.1:36869)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.864116577Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"io task","client":"1","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.864142573Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"io task","client":"2","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.864665986Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"0","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.864700673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"1","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.865433951Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"1","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.865461567Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"0","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.86607626Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:32.866109387Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.865166916Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"io task","client":"2","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"ds_connection connected from Ok(127.0.0.1:45542)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.865294844Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"io task","client":"2","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.866451053Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"2","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867048682Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"2","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"[0]R flush_numbers[0..12]: [137, 135, 140, 135, 133, 140, 132, 140, 140, 137, 139, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867625257Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"[0]R generation[0..12]: [85, 83, 86, 83, 81, 86, 80, 86, 86, 85, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867655983Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867675007Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"[1]R flush_numbers[0..12]: [137, 135, 140, 135, 133, 140, 132, 140, 140, 137, 139, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867692211Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"[1]R generation[0..12]: [85, 83, 86, 83, 81, 86, 80, 86, 86, 85, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867709095Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867726709Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"[2]R flush_numbers[0..12]: [137, 135, 140, 135, 133, 140, 132, 140, 140, 137, 139, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867743952Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"[2]R generation[0..12]: [85, 83, 86, 83, 81, 86, 80, 86, 86, 85, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867760346Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.86777708Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867792923Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867809327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Next flush: 142","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.8678249Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867840703Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"":"downstairs","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867856587Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867881072Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"0","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867894965Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"1","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867907698Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"client":"2","":"downstairs","session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867920571Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867932453Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"13e28404-eb9f-4553-8f96-65b8d3daf84d is now active with session: 58f5ef2c-a958-4c2c-9fb7-74b343975f04","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867944876Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867956558Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540,"session_id":"58f5ef2c-a958-4c2c-9fb7-74b343975f04"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867968401Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540} {"msg":"Activated sub_volume 13e28404-eb9f-4553-8f96-65b8d3daf84d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:42.867979393Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 521 len 16384 data: 7 8 9 11 02/30 Read block 372 len 16384 03/30 Read block 242 len 16384 04/30 Read block 119 len 36864 05/30 Read block 449 len 8192 06/30 Write block 530 len 24576 data: 6 5 4 6 5 5 07/30 Flush 08/30 Write block 447 len 12288 data: 9 6 6 09/30 Read block 9 len 32768 10/30 Read block 17 len 4096 11/30 Read block 241 len 12288 12/30 Read block 72 len 24576 13/30 Read block 205 len 8192 14/30 Write block 306 len 12288 data: 7 7 8 15/30 Read block 169 len 36864 16/30 Write block 544 len 8192 data: 11 10 17/30 Read block 585 len 32768 18/30 Write block 358 len 24576 data: 8 8 10 7 8 8 19/30 Write block 574 len 4096 data: 8 20/30 Read block 560 len 16384 21/30 Read block 315 len 36864 22/30 Read block 406 len 40960 23/30 Read block 62 len 20480 24/30 Write block 114 len 8192 data: 7 9 25/30 Write block 43 len 20480 data: 10 13 9 10 10 26/30 Flush 27/30 Write block 221 len 16384 data: 7 6 7 9 28/30 Write block 10 len 32768 data: 8 7 9 6 5 4 5 4 29/30 Write block 386 len 24576 data: 8 7 7 8 8 4 30/30 Write block 562 len 28672 data: 11 10 9 8 6 9 10 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 8 Sent Done Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 3 8 Done 1 2 1 4 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:42.887613449Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1540} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 85 85 85 137 137 137 T T T 1 020-039 83 83 83 135 135 135 F F F 2 040-059 88 88 88 143 143 143 F F F 3 060-079 83 83 83 135 135 135 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 88 88 88 143 143 143 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 86 86 86 140 140 140 F F F 8 160-179 86 86 86 140 140 140 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 86 86 86 139 139 139 F F F 11 220-239 86 86 86 140 140 140 T T T 12 240-259 86 86 86 140 140 140 F F F 13 260-279 86 86 86 140 140 140 F F F 14 280-299 79 79 79 127 127 127 F F F 15 300-319 88 88 88 143 143 143 F F F 16 320-339 86 86 86 139 139 139 F F F 17 340-359 88 88 88 143 143 143 F F F 18 360-379 88 88 88 143 143 143 F F F 19 380-399 86 86 86 140 140 140 T T T 20 400-419 85 85 85 137 137 137 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 88 88 88 143 143 143 F F F 23 460-479 87 87 87 141 141 141 F F F 24 480-499 84 84 84 136 136 136 F F F 25 500-519 78 78 78 126 126 126 F F F 26 520-539 88 88 88 142 142 142 F F F 27 540-559 88 88 88 143 143 143 F F F 28 560-579 88 88 88 143 143 143 T T T 29 580-599 81 81 81 133 133 133 F F F Max gen: 88, Max flush: 143 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 144 144 144 F F F 1 020-039 83 83 83 135 135 135 F F F 2 040-059 88 88 88 143 143 143 F F F 3 060-079 83 83 83 135 135 135 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 88 88 88 143 143 143 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 86 86 86 140 140 140 F F F 8 160-179 86 86 86 140 140 140 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 86 86 86 139 139 139 F F F 11 220-239 89 89 89 144 144 144 F F F 12 240-259 86 86 86 140 140 140 F F F 13 260-279 86 86 86 140 140 140 F F F 14 280-299 79 79 79 127 127 127 F F F 15 300-319 88 88 88 143 143 143 F F F 16 320-339 86 86 86 139 139 139 F F F 17 340-359 88 88 88 143 143 143 F F F 18 360-379 88 88 88 143 143 143 F F F 19 380-399 89 89 89 144 144 144 F F F 20 400-419 85 85 85 137 137 137 F F F 21 420-439 84 84 84 136 136 136 F F F 22 440-459 88 88 88 143 143 143 F F F 23 460-479 87 87 87 141 141 141 F F F 24 480-499 84 84 84 136 136 136 F F F 25 500-519 78 78 78 126 126 126 F F F 26 520-539 88 88 88 142 142 142 F F F 27 540-559 88 88 88 143 143 143 F F F 28 560-579 89 89 89 144 144 144 F F F 29 580-599 81 81 81 133 133 133 F F F Max gen: 89, Max flush: 144 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.127455031Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:47.170115956Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.170555881Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.201689719Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.202302321Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.20234436Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.202361244Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.202379058Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548} {"msg":"Crucible 254181e7-77c9-4fa6-a1d0-9bc4e77aa914 has session id: 25a8b9bc-45f2-42dd-a36a-73fd9db13415","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203004283Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Upstairs opts: Upstairs UUID: 254181e7-77c9-4fa6-a1d0-9bc4e77aa914, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203065236Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Crucible stats registered with UUID: 254181e7-77c9-4fa6-a1d0-9bc4e77aa914","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.20308483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:47.203097643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203465712Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203489227Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548} {"msg":"254181e7-77c9-4fa6-a1d0-9bc4e77aa914 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203534007Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.20355046Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"0","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203577056Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"0","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203591079Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"1","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203604802Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"1","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203617515Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"2","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203638189Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"2","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203661815Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"io task","client":"2","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203684599Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"io task","client":"0","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203708745Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"io task","client":"1","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"ds_connection connected from Ok(127.0.0.1:51057)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203745662Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"io task","client":"0","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"ds_connection connected from Ok(127.0.0.1:54806)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203774459Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"io task","client":"2","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"ds_connection connected from Ok(127.0.0.1:39240)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.203960189Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"io task","client":"1","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.204548046Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"1","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.204576332Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"2","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.204653058Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"0","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.205247217Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"1","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.205275192Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"2","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.205290126Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"0","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.205904908Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.205930114Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"[0]R flush_numbers[0..12]: [144, 135, 143, 135, 133, 143, 132, 140, 140, 137, 139, 144]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.205945107Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"[0]R generation[0..12]: [89, 83, 88, 83, 81, 88, 80, 86, 86, 85, 86, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.20595944Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.205973353Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"[1]R flush_numbers[0..12]: [144, 135, 143, 135, 133, 143, 132, 140, 140, 137, 139, 144]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.205987026Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"[1]R generation[0..12]: [89, 83, 88, 83, 81, 88, 80, 86, 86, 85, 86, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206000569Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206014552Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"[2]R flush_numbers[0..12]: [144, 135, 143, 135, 133, 143, 132, 140, 140, 137, 139, 144]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206027825Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"[2]R generation[0..12]: [89, 83, 88, 83, 81, 88, 80, 86, 86, 85, 86, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206040938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206058822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206071864Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206084567Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.20609687Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206109433Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"":"downstairs","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206122355Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206133958Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"0","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206147841Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"1","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206160664Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"client":"2","":"downstairs","session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206173686Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206185459Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"254181e7-77c9-4fa6-a1d0-9bc4e77aa914 is now active with session: 25a8b9bc-45f2-42dd-a36a-73fd9db13415","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206197782Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:24:47.206209224Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548,"session_id":"25a8b9bc-45f2-42dd-a36a-73fd9db13415"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206228538Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548} {"msg":"Activated sub_volume 254181e7-77c9-4fa6-a1d0-9bc4e77aa914","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:47.206239381Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 467 len 36864 data: 12 13 11 9 8 9 10 10 9 02/30 Write block 276 len 28672 data: 7 5 6 4 6 7 7 03/30 Flush 04/30 Flush 05/30 Write block 246 len 16384 data: 5 5 5 7 06/30 Read block 540 len 32768 07/30 Flush 08/30 Write block 58 len 4096 data: 10 09/30 Read block 520 len 24576 10/30 Write block 517 len 12288 data: 12 12 9 11/30 Flush 12/30 Write block 279 len 40960 data: 5 7 8 8 8 9 10 9 6 6 13/30 Write block 532 len 16384 data: 5 7 6 6 14/30 Read block 108 len 20480 15/30 Read block 264 len 40960 16/30 Write block 143 len 20480 data: 7 6 6 5 7 17/30 Write block 393 len 28672 data: 8 7 7 8 8 8 9 18/30 Read block 151 len 36864 19/30 Read block 381 len 24576 20/30 Write block 413 len 32768 data: 5 6 4 6 5 5 5 5 21/30 Read block 140 len 24576 22/30 Read block 415 len 24576 23/30 Write block 342 len 40960 data: 8 5 5 4 3 3 5 5 6 6 24/30 Write block 577 len 16384 data: 7 8 8 6 25/30 Write block 243 len 36864 data: 3 3 3 6 6 6 8 8 9 26/30 Write block 458 len 20480 data: 7 7 7 7 8 27/30 Read block 325 len 16384 28/30 Write block 373 len 12288 data: 6 6 8 29/30 Read block 268 len 4096 30/30 Read block 435 len 8192 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 10 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 19 19 18 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:47.226947359Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1548} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 144 144 144 F F F 1 020-039 83 83 83 135 135 135 F F F 2 040-059 90 90 90 148 148 148 F F F 3 060-079 83 83 83 135 135 135 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 88 88 88 143 143 143 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 86 86 86 140 140 140 T T T 8 160-179 86 86 86 140 140 140 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 86 86 86 139 139 139 F F F 11 220-239 89 89 89 144 144 144 F F F 12 240-259 90 90 90 147 147 147 T T T 13 260-279 90 90 90 145 145 145 T T T 14 280-299 90 90 90 145 145 145 T T T 15 300-319 88 88 88 143 143 143 F F F 16 320-339 86 86 86 139 139 139 F F F 17 340-359 88 88 88 143 143 143 T T T 18 360-379 88 88 88 143 143 143 T T T 19 380-399 89 89 89 144 144 144 T T T 20 400-419 85 85 85 137 137 137 T T T 21 420-439 84 84 84 136 136 136 T T T 22 440-459 88 88 88 143 143 143 T T T 23 460-479 90 90 90 145 145 145 T T T 24 480-499 84 84 84 136 136 136 F F F 25 500-519 90 90 90 148 148 148 F F F 26 520-539 88 88 88 142 142 142 T T T 27 540-559 88 88 88 143 143 143 F F F 28 560-579 89 89 89 144 144 144 T T T 29 580-599 81 81 81 133 133 133 T T T Max gen: 90, Max flush: 148 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 144 144 144 F F F 1 020-039 83 83 83 135 135 135 F F F 2 040-059 90 90 90 148 148 148 F F F 3 060-079 83 83 83 135 135 135 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 88 88 88 143 143 143 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 91 91 91 149 149 149 F F F 8 160-179 86 86 86 140 140 140 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 86 86 86 139 139 139 F F F 11 220-239 89 89 89 144 144 144 F F F 12 240-259 91 91 91 149 149 149 F F F 13 260-279 91 91 91 149 149 149 F F F 14 280-299 91 91 91 149 149 149 F F F 15 300-319 88 88 88 143 143 143 F F F 16 320-339 86 86 86 139 139 139 F F F 17 340-359 91 91 91 149 149 149 F F F 18 360-379 91 91 91 149 149 149 F F F 19 380-399 91 91 91 149 149 149 F F F 20 400-419 91 91 91 149 149 149 F F F 21 420-439 91 91 91 149 149 149 F F F 22 440-459 91 91 91 149 149 149 F F F 23 460-479 91 91 91 149 149 149 F F F 24 480-499 84 84 84 136 136 136 F F F 25 500-519 90 90 90 148 148 148 F F F 26 520-539 91 91 91 149 149 149 F F F 27 540-559 88 88 88 143 143 143 F F F 28 560-579 91 91 91 149 149 149 F F F 29 580-599 91 91 91 149 149 149 F F F Max gen: 91, Max flush: 149 Begin loop 46 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.544825539Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.5892761Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.589976201Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.590022351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.590042125Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.590057349Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556} {"msg":"Crucible f1fe56f2-e14d-4743-a911-d3da7f4eba18 has session id: 54eecf88-ed58-4718-af2f-812aca88eabd","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.590657848Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Upstairs opts: Upstairs UUID: f1fe56f2-e14d-4743-a911-d3da7f4eba18, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.590702588Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Crucible stats registered with UUID: f1fe56f2-e14d-4743-a911-d3da7f4eba18","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.590721172Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:51.590733704Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591271451Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591296346Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556} {"msg":"f1fe56f2-e14d-4743-a911-d3da7f4eba18 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591380844Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591399428Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"0","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591427184Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"0","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591440667Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"1","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.59145445Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"1","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591467633Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"2","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.59159311Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"2","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591627387Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"io task","client":"2","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591648792Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"io task","client":"0","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591674278Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"io task","client":"1","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"ds_connection connected from Ok(127.0.0.1:63685)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591699863Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"io task","client":"2","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"ds_connection connected from Ok(127.0.0.1:44812)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591723958Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"io task","client":"0","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"ds_connection connected from Ok(127.0.0.1:52603)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.591748454Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"io task","client":"1","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.592356855Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"0","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.592387672Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"1","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.592940631Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"1","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.593061697Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"0","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.593518705Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.593579198Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.593901158Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"2","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.594426081Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"2","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"[0]R flush_numbers[0..12]: [144, 135, 148, 135, 133, 143, 132, 149, 140, 137, 139, 144]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.594939582Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"[0]R generation[0..12]: [89, 83, 90, 83, 81, 88, 80, 91, 86, 85, 86, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.594964387Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.59497922Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"[1]R flush_numbers[0..12]: [144, 135, 148, 135, 133, 143, 132, 149, 140, 137, 139, 144]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.594994464Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"[1]R generation[0..12]: [89, 83, 90, 83, 81, 88, 80, 91, 86, 85, 86, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595008857Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.59502278Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"[2]R flush_numbers[0..12]: [144, 135, 148, 135, 133, 143, 132, 149, 140, 137, 139, 144]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595036303Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"[2]R generation[0..12]: [89, 83, 90, 83, 81, 88, 80, 91, 86, 85, 86, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595049606Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595063149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595075741Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595088284Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595100857Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.59511312Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"":"downstairs","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:24:51.595126872Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595148537Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"0","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.59516295Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"1","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595176483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"client":"2","":"downstairs","session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595189626Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595201619Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"f1fe56f2-e14d-4743-a911-d3da7f4eba18 is now active with session: 54eecf88-ed58-4718-af2f-812aca88eabd","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595213961Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595225464Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556,"session_id":"54eecf88-ed58-4718-af2f-812aca88eabd"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595237136Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556} {"msg":"Activated sub_volume f1fe56f2-e14d-4743-a911-d3da7f4eba18","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:51.595247889Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 475 len 36864 data: 10 7 6 6 6 6 6 7 9 02/30 Read block 470 len 28672 03/30 Read block 500 len 32768 04/30 Flush 05/30 Write block 288 len 40960 data: 7 5 8 5 4 4 6 6 9 10 06/30 Read block 315 len 4096 07/30 Write block 332 len 16384 data: 9 10 12 13 08/30 Read block 468 len 32768 09/30 Write block 569 len 20480 data: 10 9 10 9 10 10/30 Write block 254 len 8192 data: 8 7 11/30 Write block 71 len 28672 data: 9 11 10 13 12 10 8 12/30 Write block 563 len 4096 data: 11 13/30 Write block 140 len 40960 data: 7 6 7 8 7 7 6 8 8 8 14/30 Write block 7 len 16384 data: 11 9 7 9 15/30 Write block 209 len 40960 data: 9 8 8 7 5 4 4 4 5 6 16/30 Read block 569 len 24576 17/30 Read block 159 len 36864 18/30 Read block 237 len 40960 19/30 Read block 467 len 4096 20/30 Flush 21/30 Write block 411 len 12288 data: 6 6 6 22/30 Read block 366 len 24576 23/30 Read block 263 len 12288 24/30 Write block 169 len 4096 data: 9 25/30 Flush 26/30 Write block 303 len 4096 data: 9 27/30 Read block 86 len 4096 28/30 Read block 563 len 12288 29/30 Read block 317 len 8192 30/30 Read block 568 len 16384 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:51.615380833Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1556} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 151 151 151 F F F 1 020-039 83 83 83 135 135 135 F F F 2 040-059 90 90 90 148 148 148 F F F 3 060-079 92 92 92 151 151 151 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 88 88 88 143 143 143 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 92 92 92 151 151 151 F F F 8 160-179 92 92 92 152 152 152 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 92 92 92 151 151 151 F F F 11 220-239 89 89 89 144 144 144 F F F 12 240-259 92 92 92 151 151 151 F F F 13 260-279 91 91 91 149 149 149 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 88 88 88 143 143 143 T T T 16 320-339 92 92 92 151 151 151 F F F 17 340-359 91 91 91 149 149 149 F F F 18 360-379 91 91 91 149 149 149 F F F 19 380-399 91 91 91 149 149 149 F F F 20 400-419 92 92 92 152 152 152 F F F 21 420-439 91 91 91 149 149 149 F F F 22 440-459 91 91 91 149 149 149 F F F 23 460-479 92 92 92 150 150 150 F F F 24 480-499 92 92 92 150 150 150 F F F 25 500-519 90 90 90 148 148 148 F F F 26 520-539 91 91 91 149 149 149 F F F 27 540-559 88 88 88 143 143 143 F F F 28 560-579 92 92 92 151 151 151 F F F 29 580-599 91 91 91 149 149 149 F F F Max gen: 92, Max flush: 152 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 151 151 151 F F F 1 020-039 83 83 83 135 135 135 F F F 2 040-059 90 90 90 148 148 148 F F F 3 060-079 92 92 92 151 151 151 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 88 88 88 143 143 143 F F F 6 120-139 80 80 80 132 132 132 F F F 7 140-159 92 92 92 151 151 151 F F F 8 160-179 92 92 92 152 152 152 F F F 9 180-199 85 85 85 137 137 137 F F F 10 200-219 92 92 92 151 151 151 F F F 11 220-239 89 89 89 144 144 144 F F F 12 240-259 92 92 92 151 151 151 F F F 13 260-279 91 91 91 149 149 149 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 93 93 93 153 153 153 F F F 16 320-339 92 92 92 151 151 151 F F F 17 340-359 91 91 91 149 149 149 F F F 18 360-379 91 91 91 149 149 149 F F F 19 380-399 91 91 91 149 149 149 F F F 20 400-419 92 92 92 152 152 152 F F F 21 420-439 91 91 91 149 149 149 F F F 22 440-459 91 91 91 149 149 149 F F F 23 460-479 92 92 92 150 150 150 F F F 24 480-499 92 92 92 150 150 150 F F F 25 500-519 90 90 90 148 148 148 F F F 26 520-539 91 91 91 149 149 149 F F F 27 540-559 88 88 88 143 143 143 F F F 28 560-579 92 92 92 151 151 151 F F F 29 580-599 91 91 91 149 149 149 F F F Max gen: 93, Max flush: 153 Begin loop 47 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.835035868Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:55.877394968Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.877753435Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.908898165Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.909635164Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.909666681Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.909679794Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.909693506Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564} {"msg":"Crucible 4ac679e1-8ad1-4c07-aa98-bc56ba0aa5ba has session id: 3659f09c-91c7-4aa5-9125-cfe6ab70968c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.910426425Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"Upstairs opts: Upstairs UUID: 4ac679e1-8ad1-4c07-aa98-bc56ba0aa5ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.910461602Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"Crucible stats registered with UUID: 4ac679e1-8ad1-4c07-aa98-bc56ba0aa5ba","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.910484387Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:55.91049845Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.910990446Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911015242Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564} {"msg":"4ac679e1-8ad1-4c07-aa98-bc56ba0aa5ba active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911084347Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911102301Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"0","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911129646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"0","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911143559Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"1","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911157192Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"1","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911169775Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"2","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911183858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"2","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911205093Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"io task","client":"2","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911232318Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"io task","client":"0","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911259304Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"io task","client":"1","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58996)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.91128418Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"io task","client":"0","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"ds_connection connected from Ok(127.0.0.1:65128)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911503757Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"io task","client":"1","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57557)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.911546156Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"io task","client":"2","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.912148416Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"1","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.912191685Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"2","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.912312382Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"0","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.912858709Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"2","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.912885185Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"1","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.91299996Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"0","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913564202Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913598389Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"[0]R flush_numbers[0..12]: [151, 135, 148, 151, 133, 143, 132, 151, 152, 137, 151, 144]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913622554Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"[0]R generation[0..12]: [92, 83, 90, 92, 81, 88, 80, 92, 92, 85, 92, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913639068Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913653281Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"[1]R flush_numbers[0..12]: [151, 135, 148, 151, 133, 143, 132, 151, 152, 137, 151, 144]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913667194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"[1]R generation[0..12]: [92, 83, 90, 92, 81, 88, 80, 92, 92, 85, 92, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913680737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.91369438Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"[2]R flush_numbers[0..12]: [151, 135, 148, 151, 133, 143, 132, 151, 152, 137, 151, 144]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913707733Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"[2]R generation[0..12]: [92, 83, 90, 92, 81, 88, 80, 92, 92, 85, 92, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913724036Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913738469Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913760244Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.91379007Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913812455Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913826248Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"":"downstairs","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913838861Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913850443Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"0","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913863306Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"1","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913875929Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"client":"2","":"downstairs","session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913889412Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913901245Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"4ac679e1-8ad1-4c07-aa98-bc56ba0aa5ba is now active with session: 3659f09c-91c7-4aa5-9125-cfe6ab70968c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913913567Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913929071Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564,"session_id":"3659f09c-91c7-4aa5-9125-cfe6ab70968c"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913940993Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564} {"msg":"Activated sub_volume 4ac679e1-8ad1-4c07-aa98-bc56ba0aa5ba","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:24:55.913951895Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 327 len 40960 data: 8 5 5 6 10 10 11 13 14 11 02/30 Read block 359 len 12288 03/30 Write block 332 len 32768 data: 11 12 14 15 12 9 9 10 04/30 Flush 05/30 Write block 378 len 24576 data: 7 9 9 10 9 9 06/30 Read block 511 len 36864 07/30 Flush 08/30 Write block 272 len 12288 data: 8 8 7 09/30 Read block 482 len 16384 10/30 Write block 200 len 40960 data: 9 11 8 8 8 10 8 7 8 10 11/30 Read block 590 len 24576 12/30 Write block 242 len 8192 data: 4 4 13/30 Write block 112 len 40960 data: 4 6 8 10 11 10 11 9 9 9 14/30 Read block 463 len 8192 15/30 Read block 106 len 28672 16/30 Read block 34 len 40960 17/30 Read block 547 len 12288 18/30 Write block 208 len 12288 data: 9 11 9 19/30 Read block 544 len 28672 20/30 Write block 493 len 40960 data: 11 9 9 8 8 8 9 9 11 10 21/30 Read block 275 len 36864 22/30 Write block 238 len 20480 data: 8 7 6 5 5 23/30 Read block 24 len 8192 24/30 Write block 193 len 32768 data: 4 6 6 5 7 6 6 10 25/30 Write block 323 len 28672 data: 8 8 9 8 9 6 6 26/30 Write block 143 len 16384 data: 9 8 8 7 27/30 Read block 356 len 8192 28/30 Write block 9 len 4096 data: 8 29/30 Write block 531 len 40960 data: 6 6 8 7 7 5 6 6 9 9 30/30 Read block 504 len 28672 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 3 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:24:55.935335429Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1564} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 151 151 151 T T T 1 020-039 83 83 83 135 135 135 F F F 2 040-059 90 90 90 148 148 148 F F F 3 060-079 92 92 92 151 151 151 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 88 88 88 143 143 143 T T T 6 120-139 80 80 80 132 132 132 T T T 7 140-159 92 92 92 151 151 151 T T T 8 160-179 92 92 92 152 152 152 F F F 9 180-199 85 85 85 137 137 137 T T T 10 200-219 92 92 92 151 151 151 T T T 11 220-239 89 89 89 144 144 144 T T T 12 240-259 92 92 92 151 151 151 T T T 13 260-279 91 91 91 149 149 149 T T T 14 280-299 92 92 92 151 151 151 F F F 15 300-319 93 93 93 153 153 153 F F F 16 320-339 94 94 94 154 154 154 T T T 17 340-359 91 91 91 149 149 149 F F F 18 360-379 94 94 94 155 155 155 F F F 19 380-399 94 94 94 155 155 155 F F F 20 400-419 92 92 92 152 152 152 F F F 21 420-439 91 91 91 149 149 149 F F F 22 440-459 91 91 91 149 149 149 F F F 23 460-479 92 92 92 150 150 150 F F F 24 480-499 92 92 92 150 150 150 T T T 25 500-519 90 90 90 148 148 148 T T T 26 520-539 91 91 91 149 149 149 T T T 27 540-559 88 88 88 143 143 143 T T T 28 560-579 92 92 92 151 151 151 F F F 29 580-599 91 91 91 149 149 149 F F F Max gen: 94, Max flush: 155 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 156 156 156 F F F 1 020-039 83 83 83 135 135 135 F F F 2 040-059 90 90 90 148 148 148 F F F 3 060-079 92 92 92 151 151 151 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 95 95 95 156 156 156 F F F 6 120-139 95 95 95 156 156 156 F F F 7 140-159 95 95 95 156 156 156 F F F 8 160-179 92 92 92 152 152 152 F F F 9 180-199 95 95 95 156 156 156 F F F 10 200-219 95 95 95 156 156 156 F F F 11 220-239 95 95 95 156 156 156 F F F 12 240-259 95 95 95 156 156 156 F F F 13 260-279 95 95 95 156 156 156 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 93 93 93 153 153 153 F F F 16 320-339 95 95 95 156 156 156 F F F 17 340-359 91 91 91 149 149 149 F F F 18 360-379 94 94 94 155 155 155 F F F 19 380-399 94 94 94 155 155 155 F F F 20 400-419 92 92 92 152 152 152 F F F 21 420-439 91 91 91 149 149 149 F F F 22 440-459 91 91 91 149 149 149 F F F 23 460-479 92 92 92 150 150 150 F F F 24 480-499 95 95 95 156 156 156 F F F 25 500-519 95 95 95 156 156 156 F F F 26 520-539 95 95 95 156 156 156 F F F 27 540-559 95 95 95 156 156 156 F F F 28 560-579 92 92 92 151 151 151 F F F 29 580-599 91 91 91 149 149 149 F F F Max gen: 95, Max flush: 156 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.249599578Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:00.292321297Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.292691226Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.323786026Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.324428764Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.324462252Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.324476364Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.324490948Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572} {"msg":"Crucible 259ba084-c9b9-4b3c-9e3e-f75ec656f506 has session id: 3c3b74c6-bacf-4d6b-9403-6c0db4654f2d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325163043Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Upstairs opts: Upstairs UUID: 259ba084-c9b9-4b3c-9e3e-f75ec656f506, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325199741Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Crucible stats registered with UUID: 259ba084-c9b9-4b3c-9e3e-f75ec656f506","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325216654Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:00.325229237Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325628843Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325651718Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572} {"msg":"259ba084-c9b9-4b3c-9e3e-f75ec656f506 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325712811Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325731205Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"0","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325761992Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"0","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325791919Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"1","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325820354Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"1","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.32584788Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"2","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325875886Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"2","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325901632Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"io task","client":"2","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325930008Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"io task","client":"0","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325959875Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"io task","client":"1","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56655)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.325988641Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"io task","client":"2","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57518)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.326014056Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"io task","client":"0","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"ds_connection connected from Ok(127.0.0.1:52459)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.326039392Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"io task","client":"1","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.326732691Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"1","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.326768039Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"2","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.326848136Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"0","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.327341003Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"1","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.327455927Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"0","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.327480213Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"2","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.327969228Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328080192Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"[0]R flush_numbers[0..12]: [156, 135, 148, 151, 133, 156, 156, 156, 152, 156, 156, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328106278Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"[0]R generation[0..12]: [95, 83, 90, 92, 81, 95, 95, 95, 92, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328121481Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328135814Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"[1]R flush_numbers[0..12]: [156, 135, 148, 151, 133, 156, 156, 156, 152, 156, 156, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328149457Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"[1]R generation[0..12]: [95, 83, 90, 92, 81, 95, 95, 95, 92, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.32816263Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328177413Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"[2]R flush_numbers[0..12]: [156, 135, 148, 151, 133, 156, 156, 156, 152, 156, 156, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328191456Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"[2]R generation[0..12]: [95, 83, 90, 92, 81, 95, 95, 95, 92, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328204769Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328232765Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328248108Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328260971Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-17T22:25:00.328277645Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328297399Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"":"downstairs","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328310992Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328322705Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"0","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328335597Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"1","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.32834822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"client":"2","":"downstairs","session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328360993Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328372855Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"259ba084-c9b9-4b3c-9e3e-f75ec656f506 is now active with session: 3c3b74c6-bacf-4d6b-9403-6c0db4654f2d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328385138Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.32839667Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572,"session_id":"3c3b74c6-bacf-4d6b-9403-6c0db4654f2d"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328408363Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572} {"msg":"Activated sub_volume 259ba084-c9b9-4b3c-9e3e-f75ec656f506","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:00.328419815Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 153 len 40960 data: 8 11 10 10 9 9 10 10 9 7 02/30 Write block 169 len 16384 data: 10 7 6 7 03/30 Write block 593 len 8192 data: 5 4 04/30 Write block 156 len 16384 data: 11 10 10 11 05/30 Flush 06/30 Write block 304 len 20480 data: 9 8 8 8 9 07/30 Read block 21 len 28672 08/30 Read block 187 len 8192 09/30 Read block 433 len 24576 10/30 Flush 11/30 Flush 12/30 Write block 300 len 4096 data: 9 13/30 Read block 491 len 24576 14/30 Read block 453 len 28672 15/30 Read block 83 len 28672 16/30 Write block 66 len 8192 data: 8 9 17/30 Read block 59 len 4096 18/30 Write block 576 len 8192 data: 6 8 19/30 Write block 35 len 24576 data: 7 6 8 10 11 8 20/30 Write block 69 len 40960 data: 9 11 10 12 11 14 13 11 9 8 21/30 Read block 520 len 36864 22/30 Write block 529 len 28672 data: 9 7 7 7 9 8 8 23/30 Read block 6 len 8192 24/30 Write block 540 len 40960 data: 10 9 11 10 12 11 11 9 8 8 25/30 Write block 216 len 20480 data: 5 6 7 7 7 26/30 Write block 346 len 8192 data: 4 4 27/30 Write block 403 len 32768 data: 7 7 7 8 11 9 7 6 28/30 Read block 383 len 4096 29/30 Read block 150 len 4096 30/30 Write block 144 len 24576 data: 9 9 8 9 9 9 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 1 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:00.346845971Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1572} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 156 156 156 F F F 1 020-039 83 83 83 135 135 135 T T T 2 040-059 90 90 90 148 148 148 T T T 3 060-079 92 92 92 151 151 151 T T T 4 080-099 81 81 81 133 133 133 F F F 5 100-119 95 95 95 156 156 156 F F F 6 120-139 95 95 95 156 156 156 F F F 7 140-159 96 96 96 157 157 157 T T T 8 160-179 96 96 96 157 157 157 F F F 9 180-199 95 95 95 156 156 156 F F F 10 200-219 95 95 95 156 156 156 T T T 11 220-239 95 95 95 156 156 156 T T T 12 240-259 95 95 95 156 156 156 F F F 13 260-279 95 95 95 156 156 156 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 96 96 96 158 158 158 T T T 16 320-339 95 95 95 156 156 156 F F F 17 340-359 91 91 91 149 149 149 T T T 18 360-379 94 94 94 155 155 155 F F F 19 380-399 94 94 94 155 155 155 F F F 20 400-419 92 92 92 152 152 152 T T T 21 420-439 91 91 91 149 149 149 F F F 22 440-459 91 91 91 149 149 149 F F F 23 460-479 92 92 92 150 150 150 F F F 24 480-499 95 95 95 156 156 156 F F F 25 500-519 95 95 95 156 156 156 F F F 26 520-539 95 95 95 156 156 156 T T T 27 540-559 95 95 95 156 156 156 T T T 28 560-579 92 92 92 151 151 151 T T T 29 580-599 96 96 96 157 157 157 F F F Max gen: 96, Max flush: 158 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 156 156 156 F F F 1 020-039 97 97 97 159 159 159 F F F 2 040-059 97 97 97 159 159 159 F F F 3 060-079 97 97 97 159 159 159 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 95 95 95 156 156 156 F F F 6 120-139 95 95 95 156 156 156 F F F 7 140-159 97 97 97 159 159 159 F F F 8 160-179 96 96 96 157 157 157 F F F 9 180-199 95 95 95 156 156 156 F F F 10 200-219 97 97 97 159 159 159 F F F 11 220-239 97 97 97 159 159 159 F F F 12 240-259 95 95 95 156 156 156 F F F 13 260-279 95 95 95 156 156 156 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 97 97 97 159 159 159 F F F 16 320-339 95 95 95 156 156 156 F F F 17 340-359 97 97 97 159 159 159 F F F 18 360-379 94 94 94 155 155 155 F F F 19 380-399 94 94 94 155 155 155 F F F 20 400-419 97 97 97 159 159 159 F F F 21 420-439 91 91 91 149 149 149 F F F 22 440-459 91 91 91 149 149 149 F F F 23 460-479 92 92 92 150 150 150 F F F 24 480-499 95 95 95 156 156 156 F F F 25 500-519 95 95 95 156 156 156 F F F 26 520-539 97 97 97 159 159 159 F F F 27 540-559 97 97 97 159 159 159 F F F 28 560-579 97 97 97 159 159 159 F F F 29 580-599 96 96 96 157 157 157 F F F Max gen: 97, Max flush: 159 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.650315891Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.696810803Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.697547983Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.697590192Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.697605155Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.697620398Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580} {"msg":"Crucible 84e10e59-9448-4c6b-b689-7a454cc1f78b has session id: 49baac9d-1c04-42f1-b4ea-4dd46236153d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.698134549Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Upstairs opts: Upstairs UUID: 84e10e59-9448-4c6b-b689-7a454cc1f78b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.698179349Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Crucible stats registered with UUID: 84e10e59-9448-4c6b-b689-7a454cc1f78b","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.698204004Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:04.698225439Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.698671785Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.698705632Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580} {"msg":"84e10e59-9448-4c6b-b689-7a454cc1f78b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.698759514Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.69878567Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"0","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.698818657Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"0","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.69883436Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"1","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.698855165Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"1","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.69887904Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"2","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.698904845Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"2","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.69892712Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"io task","client":"2","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.698948285Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"io task","client":"0","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.698970549Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"io task","client":"1","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58801)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.699004377Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"io task","client":"2","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36953)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.699033043Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"io task","client":"0","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44884)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.699060769Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"io task","client":"1","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.699728213Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"0","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.69975903Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"1","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.700405989Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"0","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.700476705Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"1","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.700619115Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"2","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.700991766Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.701118003Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.701408446Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"2","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"[0]R flush_numbers[0..12]: [156, 159, 159, 159, 133, 156, 156, 159, 157, 156, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.701936289Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"[0]R generation[0..12]: [95, 97, 97, 97, 81, 95, 95, 97, 96, 95, 97, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.701961035Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.701976618Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"[1]R flush_numbers[0..12]: [156, 159, 159, 159, 133, 156, 156, 159, 157, 156, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.701995082Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"[1]R generation[0..12]: [95, 97, 97, 97, 81, 95, 95, 97, 96, 95, 97, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702009135Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702023178Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"[2]R flush_numbers[0..12]: [156, 159, 159, 159, 133, 156, 156, 159, 157, 156, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702036621Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"[2]R generation[0..12]: [95, 97, 97, 97, 81, 95, 95, 97, 96, 95, 97, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702049574Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702063387Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.70207608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702088712Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702101345Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702113818Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"":"downstairs","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.70212647Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702137953Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"0","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702150836Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"1","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702163718Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"client":"2","":"downstairs","session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702176611Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702198316Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"84e10e59-9448-4c6b-b689-7a454cc1f78b is now active with session: 49baac9d-1c04-42f1-b4ea-4dd46236153d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702210909Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702222461Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580,"session_id":"49baac9d-1c04-42f1-b4ea-4dd46236153d"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702234384Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580} {"msg":"Activated sub_volume 84e10e59-9448-4c6b-b689-7a454cc1f78b","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:04.702245226Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 495 len 28672 data: 10 9 9 9 10 10 12 02/30 Read block 388 len 40960 03/30 Write block 370 len 8192 data: 5 5 04/30 Write block 387 len 8192 data: 8 8 05/30 Write block 43 len 4096 data: 11 06/30 Read block 34 len 32768 07/30 Read block 1 len 36864 08/30 Read block 238 len 40960 09/30 Read block 372 len 4096 10/30 Write block 428 len 36864 data: 9 8 9 8 8 8 7 6 6 11/30 Read block 373 len 40960 12/30 Write block 151 len 20480 data: 10 9 9 12 11 13/30 Read block 38 len 12288 14/30 Read block 381 len 12288 15/30 Write block 243 len 36864 data: 5 4 4 7 7 7 9 9 10 16/30 Write block 262 len 12288 data: 7 10 11 17/30 Read block 430 len 8192 18/30 Write block 550 len 4096 data: 8 19/30 Write block 554 len 16384 data: 9 8 9 8 20/30 Read block 499 len 16384 21/30 Write block 491 len 24576 data: 14 14 12 10 11 10 22/30 Flush 23/30 Write block 557 len 24576 data: 9 10 9 11 10 12 24/30 Read block 50 len 4096 25/30 Read block 167 len 8192 26/30 Read block 370 len 36864 27/30 Read block 470 len 20480 28/30 Read block 148 len 8192 29/30 Read block 192 len 40960 30/30 Write block 112 len 40960 data: 5 7 9 11 12 11 12 10 10 10 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:04.723170521Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1580} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 156 156 156 F F F 1 020-039 97 97 97 159 159 159 F F F 2 040-059 98 98 98 160 160 160 F F F 3 060-079 97 97 97 159 159 159 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 95 95 95 156 156 156 T T T 6 120-139 95 95 95 156 156 156 T T T 7 140-159 98 98 98 160 160 160 F F F 8 160-179 96 96 96 157 157 157 F F F 9 180-199 95 95 95 156 156 156 F F F 10 200-219 97 97 97 159 159 159 F F F 11 220-239 97 97 97 159 159 159 F F F 12 240-259 98 98 98 160 160 160 F F F 13 260-279 98 98 98 160 160 160 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 97 97 97 159 159 159 F F F 16 320-339 95 95 95 156 156 156 F F F 17 340-359 97 97 97 159 159 159 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 98 98 98 160 160 160 F F F 20 400-419 97 97 97 159 159 159 F F F 21 420-439 98 98 98 160 160 160 F F F 22 440-459 91 91 91 149 149 149 F F F 23 460-479 92 92 92 150 150 150 F F F 24 480-499 98 98 98 160 160 160 F F F 25 500-519 98 98 98 160 160 160 F F F 26 520-539 97 97 97 159 159 159 F F F 27 540-559 98 98 98 160 160 160 T T T 28 560-579 97 97 97 159 159 159 T T T 29 580-599 96 96 96 157 157 157 F F F Max gen: 98, Max flush: 160 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 156 156 156 F F F 1 020-039 97 97 97 159 159 159 F F F 2 040-059 98 98 98 160 160 160 F F F 3 060-079 97 97 97 159 159 159 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 99 99 99 161 161 161 F F F 6 120-139 99 99 99 161 161 161 F F F 7 140-159 98 98 98 160 160 160 F F F 8 160-179 96 96 96 157 157 157 F F F 9 180-199 95 95 95 156 156 156 F F F 10 200-219 97 97 97 159 159 159 F F F 11 220-239 97 97 97 159 159 159 F F F 12 240-259 98 98 98 160 160 160 F F F 13 260-279 98 98 98 160 160 160 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 97 97 97 159 159 159 F F F 16 320-339 95 95 95 156 156 156 F F F 17 340-359 97 97 97 159 159 159 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 98 98 98 160 160 160 F F F 20 400-419 97 97 97 159 159 159 F F F 21 420-439 98 98 98 160 160 160 F F F 22 440-459 91 91 91 149 149 149 F F F 23 460-479 92 92 92 150 150 150 F F F 24 480-499 98 98 98 160 160 160 F F F 25 500-519 98 98 98 160 160 160 F F F 26 520-539 97 97 97 159 159 159 F F F 27 540-559 99 99 99 161 161 161 F F F 28 560-579 99 99 99 161 161 161 F F F 29 580-599 96 96 96 157 157 157 F F F Max gen: 99, Max flush: 161 Begin loop 50 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:08.965905817Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.010137411Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.010886652Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.010926821Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.010940854Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.010954927Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588} {"msg":"Crucible b6013f2c-9f97-414f-932b-07a282d9b04c has session id: 920b998f-7815-4328-b1f1-6be5a181fdaf","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.011593525Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Upstairs opts: Upstairs UUID: b6013f2c-9f97-414f-932b-07a282d9b04c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.011638965Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Crucible stats registered with UUID: b6013f2c-9f97-414f-932b-07a282d9b04c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.0116646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:09.011685875Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012159887Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012193734Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588} {"msg":"b6013f2c-9f97-414f-932b-07a282d9b04c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012247436Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012276872Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"0","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012323282Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"0","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012351608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"1","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012384395Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"1","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012412201Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"2","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012440078Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"2","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012485337Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"io task","client":"2","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012502301Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"io task","client":"0","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012516144Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"io task","client":"1","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"ds_connection connected from Ok(127.0.0.1:57178)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012530707Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"io task","client":"2","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"ds_connection connected from Ok(127.0.0.1:42916)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.012773659Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"io task","client":"1","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"ds_connection connected from Ok(127.0.0.1:58082)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.01282154Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"io task","client":"0","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.013220496Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"2","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.013372619Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"0","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.013792009Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"2","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.013936921Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"0","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.01426104Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"1","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.014479017Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.014498722Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.014839585Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"1","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"[0]R flush_numbers[0..12]: [156, 159, 160, 159, 133, 161, 161, 160, 157, 156, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015349605Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"[0]R generation[0..12]: [95, 97, 98, 97, 81, 99, 99, 98, 96, 95, 97, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.01537388Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015389134Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"[1]R flush_numbers[0..12]: [156, 159, 160, 159, 133, 161, 161, 160, 157, 156, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015402937Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"[1]R generation[0..12]: [95, 97, 98, 97, 81, 99, 99, 98, 96, 95, 97, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.0154162Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015429903Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"[2]R flush_numbers[0..12]: [156, 159, 160, 159, 133, 161, 161, 160, 157, 156, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015443586Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"[2]R generation[0..12]: [95, 97, 98, 97, 81, 99, 99, 98, 96, 95, 97, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015456558Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015470091Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015482624Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015495607Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.01551219Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015531774Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"":"downstairs","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015545248Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.01555705Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"0","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015570013Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"1","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015582865Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"client":"2","":"downstairs","session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015595758Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015607591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"b6013f2c-9f97-414f-932b-07a282d9b04c is now active with session: 920b998f-7815-4328-b1f1-6be5a181fdaf","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015620094Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015631706Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588,"session_id":"920b998f-7815-4328-b1f1-6be5a181fdaf"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015643609Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588} {"msg":"Activated sub_volume b6013f2c-9f97-414f-932b-07a282d9b04c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:09.015654431Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 64 len 36864 data: 9 8 9 10 9 10 12 11 13 02/30 Write block 394 len 32768 data: 8 8 9 9 9 10 8 7 03/30 Write block 49 len 28672 data: 14 12 10 11 11 11 10 04/30 Read block 447 len 32768 05/30 Write block 163 len 40960 data: 10 13 17 14 9 8 11 8 7 8 06/30 Flush 07/30 Write block 118 len 28672 data: 13 11 11 11 11 11 7 08/30 Write block 344 len 16384 data: 6 5 5 5 09/30 Read block 39 len 8192 10/30 Write block 44 len 20480 data: 14 10 11 11 13 11/30 Write block 405 len 8192 data: 8 9 12/30 Write block 244 len 8192 data: 5 5 13/30 Read block 22 len 32768 14/30 Write block 210 len 12288 data: 10 9 8 15/30 Read block 151 len 20480 16/30 Read block 529 len 24576 17/30 Write block 575 len 20480 data: 9 7 9 9 9 18/30 Read block 266 len 4096 19/30 Write block 34 len 32768 data: 5 8 7 9 11 12 9 8 20/30 Read block 226 len 4096 21/30 Read block 514 len 16384 22/30 Write block 524 len 32768 data: 12 11 12 12 10 10 8 8 23/30 Write block 564 len 32768 data: 10 9 7 10 11 11 10 11 24/30 Read block 19 len 12288 25/30 Read block 316 len 40960 26/30 Read block 140 len 32768 27/30 Write block 302 len 12288 data: 10 10 10 28/30 Write block 531 len 28672 data: 9 8 10 9 9 6 7 29/30 Write block 111 len 24576 data: 4 6 8 10 12 13 30/30 Read block 368 len 32768 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 7 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 24 23 22 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:09.036080658Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1588} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 156 156 156 F F F 1 020-039 97 97 97 159 159 159 T T T 2 040-059 100 100 100 162 162 162 T T T 3 060-079 100 100 100 162 162 162 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 99 99 99 161 161 161 T T T 6 120-139 99 99 99 161 161 161 T T T 7 140-159 98 98 98 160 160 160 F F F 8 160-179 100 100 100 162 162 162 F F F 9 180-199 95 95 95 156 156 156 F F F 10 200-219 97 97 97 159 159 159 T T T 11 220-239 97 97 97 159 159 159 F F F 12 240-259 98 98 98 160 160 160 T T T 13 260-279 98 98 98 160 160 160 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 97 97 97 159 159 159 T T T 16 320-339 95 95 95 156 156 156 F F F 17 340-359 97 97 97 159 159 159 T T T 18 360-379 98 98 98 160 160 160 F F F 19 380-399 100 100 100 162 162 162 F F F 20 400-419 100 100 100 162 162 162 T T T 21 420-439 98 98 98 160 160 160 F F F 22 440-459 91 91 91 149 149 149 F F F 23 460-479 92 92 92 150 150 150 F F F 24 480-499 98 98 98 160 160 160 F F F 25 500-519 98 98 98 160 160 160 F F F 26 520-539 97 97 97 159 159 159 T T T 27 540-559 99 99 99 161 161 161 F F F 28 560-579 99 99 99 161 161 161 T T T 29 580-599 96 96 96 157 157 157 F F F Max gen: 100, Max flush: 162 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 156 156 156 F F F 1 020-039 101 101 101 163 163 163 F F F 2 040-059 101 101 101 163 163 163 F F F 3 060-079 100 100 100 162 162 162 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 101 101 101 163 163 163 F F F 6 120-139 101 101 101 163 163 163 F F F 7 140-159 98 98 98 160 160 160 F F F 8 160-179 100 100 100 162 162 162 F F F 9 180-199 95 95 95 156 156 156 F F F 10 200-219 101 101 101 163 163 163 F F F 11 220-239 97 97 97 159 159 159 F F F 12 240-259 101 101 101 163 163 163 F F F 13 260-279 98 98 98 160 160 160 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 101 101 101 163 163 163 F F F 16 320-339 95 95 95 156 156 156 F F F 17 340-359 101 101 101 163 163 163 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 100 100 100 162 162 162 F F F 20 400-419 101 101 101 163 163 163 F F F 21 420-439 98 98 98 160 160 160 F F F 22 440-459 91 91 91 149 149 149 F F F 23 460-479 92 92 92 150 150 150 F F F 24 480-499 98 98 98 160 160 160 F F F 25 500-519 98 98 98 160 160 160 F F F 26 520-539 101 101 101 163 163 163 F F F 27 540-559 99 99 99 161 161 161 F F F 28 560-579 101 101 101 163 163 163 F F F 29 580-599 96 96 96 157 157 157 F F F Max gen: 101, Max flush: 163 Begin loop 51 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.309493244Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.353723007Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.354423189Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.354465968Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.354481561Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.354495874Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596} {"msg":"Crucible 1cff53b5-9734-4eae-8422-27ec541030ba has session id: df941dfe-5b5f-4116-adb2-c8efe803bf44","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355094263Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Upstairs opts: Upstairs UUID: 1cff53b5-9734-4eae-8422-27ec541030ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355130161Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Crucible stats registered with UUID: 1cff53b5-9734-4eae-8422-27ec541030ba","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355146325Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:23.355159177Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.35554416Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355577298Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596} {"msg":"1cff53b5-9734-4eae-8422-27ec541030ba active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355633339Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355659835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"0","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355689042Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"0","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355704235Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"1","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355719188Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"1","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355732221Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"2","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355745984Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"2","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355758807Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"io task","client":"2","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355772109Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"io task","client":"0","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355793444Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"io task","client":"1","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"ds_connection connected from Ok(127.0.0.1:51292)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355828352Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"io task","client":"2","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"ds_connection connected from Ok(127.0.0.1:61460)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355904308Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"io task","client":"0","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"ds_connection connected from Ok(127.0.0.1:33834)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.355948038Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"io task","client":"1","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.356520311Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"1","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.356668173Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"0","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.357167751Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"1","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.357309891Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"0","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.357789585Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.357900189Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.358145592Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"2","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.358618043Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"2","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"[0]R flush_numbers[0..12]: [156, 163, 163, 162, 133, 163, 163, 160, 162, 156, 163, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359144857Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"[0]R generation[0..12]: [95, 101, 101, 100, 81, 101, 101, 98, 100, 95, 101, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359164801Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359183225Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"[1]R flush_numbers[0..12]: [156, 163, 163, 162, 133, 163, 163, 160, 162, 156, 163, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359196968Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"[1]R generation[0..12]: [95, 101, 101, 100, 81, 101, 101, 98, 100, 95, 101, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359210671Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359224044Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"[2]R flush_numbers[0..12]: [156, 163, 163, 162, 133, 163, 163, 160, 162, 156, 163, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359238117Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"[2]R generation[0..12]: [95, 101, 101, 100, 81, 101, 101, 98, 100, 95, 101, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.35925205Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359265613Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359279326Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359301471Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:25:23.359325326Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359350652Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"":"downstairs","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359376127Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359392901Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"0","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359406153Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"1","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359418896Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"client":"2","":"downstairs","session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359443442Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359467707Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"1cff53b5-9734-4eae-8422-27ec541030ba is now active with session: df941dfe-5b5f-4116-adb2-c8efe803bf44","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359492252Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359507195Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596,"session_id":"df941dfe-5b5f-4116-adb2-c8efe803bf44"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359519118Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596} {"msg":"Activated sub_volume 1cff53b5-9734-4eae-8422-27ec541030ba","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:23.359537442Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 309 len 36864 data: 8 5 6 6 7 6 6 8 8 03/30 Write block 148 len 28672 data: 10 10 8 11 10 10 13 04/30 Write block 426 len 36864 data: 7 8 10 9 10 9 9 9 8 05/30 Flush 06/30 Write block 517 len 28672 data: 13 13 10 7 8 9 10 07/30 Read block 218 len 16384 08/30 Read block 412 len 20480 09/30 Write block 211 len 4096 data: 10 10/30 Flush 11/30 Flush 12/30 Write block 453 len 24576 data: 12 10 8 6 8 8 13/30 Read block 300 len 20480 14/30 Write block 553 len 12288 data: 9 10 9 15/30 Read block 426 len 8192 16/30 Write block 566 len 16384 data: 8 11 12 12 17/30 Write block 301 len 24576 data: 10 11 11 11 9 9 18/30 Read block 573 len 12288 19/30 Read block 113 len 12288 20/30 Flush 21/30 Write block 152 len 8192 data: 11 11 22/30 Read block 494 len 36864 23/30 Read block 461 len 24576 24/30 Read block 380 len 24576 25/30 Read block 1 len 16384 26/30 Read block 399 len 36864 27/30 Read block 152 len 20480 28/30 Read block 36 len 20480 29/30 Read block 464 len 32768 30/30 Read block 47 len 20480 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 10 9 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:23.379832241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1596} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 156 156 156 F F F 1 020-039 101 101 101 163 163 163 F F F 2 040-059 101 101 101 163 163 163 F F F 3 060-079 100 100 100 162 162 162 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 101 101 101 163 163 163 F F F 6 120-139 101 101 101 163 163 163 F F F 7 140-159 102 102 102 165 165 165 T T T 8 160-179 100 100 100 162 162 162 F F F 9 180-199 95 95 95 156 156 156 F F F 10 200-219 102 102 102 166 166 166 F F F 11 220-239 97 97 97 159 159 159 F F F 12 240-259 101 101 101 163 163 163 F F F 13 260-279 98 98 98 160 160 160 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 102 102 102 168 168 168 F F F 16 320-339 95 95 95 156 156 156 F F F 17 340-359 101 101 101 163 163 163 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 100 100 100 162 162 162 F F F 20 400-419 101 101 101 163 163 163 F F F 21 420-439 102 102 102 165 165 165 F F F 22 440-459 102 102 102 168 168 168 F F F 23 460-479 92 92 92 150 150 150 F F F 24 480-499 98 98 98 160 160 160 F F F 25 500-519 102 102 102 166 166 166 F F F 26 520-539 102 102 102 166 166 166 F F F 27 540-559 102 102 102 168 168 168 F F F 28 560-579 102 102 102 168 168 168 F F F 29 580-599 96 96 96 157 157 157 F F F Max gen: 102, Max flush: 168 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 156 156 156 F F F 1 020-039 101 101 101 163 163 163 F F F 2 040-059 101 101 101 163 163 163 F F F 3 060-079 100 100 100 162 162 162 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 101 101 101 163 163 163 F F F 6 120-139 101 101 101 163 163 163 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 100 100 100 162 162 162 F F F 9 180-199 95 95 95 156 156 156 F F F 10 200-219 102 102 102 166 166 166 F F F 11 220-239 97 97 97 159 159 159 F F F 12 240-259 101 101 101 163 163 163 F F F 13 260-279 98 98 98 160 160 160 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 102 102 102 168 168 168 F F F 16 320-339 95 95 95 156 156 156 F F F 17 340-359 101 101 101 163 163 163 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 100 100 100 162 162 162 F F F 20 400-419 101 101 101 163 163 163 F F F 21 420-439 102 102 102 165 165 165 F F F 22 440-459 102 102 102 168 168 168 F F F 23 460-479 92 92 92 150 150 150 F F F 24 480-499 98 98 98 160 160 160 F F F 25 500-519 102 102 102 166 166 166 F F F 26 520-539 102 102 102 166 166 166 F F F 27 540-559 102 102 102 168 168 168 F F F 28 560-579 102 102 102 168 168 168 F F F 29 580-599 96 96 96 157 157 157 F F F Max gen: 103, Max flush: 169 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.600345732Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:27.643569389Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.643940999Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.675217948Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.67583337Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.675868508Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.675882221Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.675896264Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604} {"msg":"Crucible 63049b96-104d-4649-bd5a-4614367edf44 has session id: 8b390bf9-9a20-4516-b216-337ca4dc91cb","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.676496043Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Upstairs opts: Upstairs UUID: 63049b96-104d-4649-bd5a-4614367edf44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.676534912Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Crucible stats registered with UUID: 63049b96-104d-4649-bd5a-4614367edf44","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.676560457Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:27.676578421Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677062466Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.67708616Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604} {"msg":"63049b96-104d-4649-bd5a-4614367edf44 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677166258Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677185042Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"0","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677219359Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"0","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677244535Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"1","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.67726941Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"1","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677293015Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"2","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677319051Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"2","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677345416Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"io task","client":"2","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677372852Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"io task","client":"0","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48727)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677402079Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"io task","client":"0","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677431615Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"io task","client":"1","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:46712)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677460741Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"io task","client":"2","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:46061)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.677488918Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"io task","client":"1","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.678164883Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"2","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.678192349Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"0","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.678207933Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"1","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.678825836Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"0","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.678959805Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"1","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.678993592Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"2","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679472035Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679582779Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"[0]R flush_numbers[0..12]: [156, 163, 163, 162, 133, 163, 163, 169, 162, 156, 166, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679612286Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"[0]R generation[0..12]: [95, 101, 101, 100, 81, 101, 101, 103, 100, 95, 102, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679627209Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679641082Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"[1]R flush_numbers[0..12]: [156, 163, 163, 162, 133, 163, 163, 169, 162, 156, 166, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679654605Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"[1]R generation[0..12]: [95, 101, 101, 100, 81, 101, 101, 103, 100, 95, 102, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679667908Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.67968158Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"[2]R flush_numbers[0..12]: [156, 163, 163, 162, 133, 163, 163, 169, 162, 156, 166, 159]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679695223Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"[2]R generation[0..12]: [95, 101, 101, 100, 81, 101, 101, 103, 100, 95, 102, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679708376Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679721909Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679738863Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679752366Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679764829Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679777071Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"":"downstairs","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679797445Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679812029Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604Wait for a query_work_queue command to finish before sending IO ,"client":"0","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679839145Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"1","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679853028Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"client":"2","":"downstairs","session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.67986585Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679877713Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"63049b96-104d-4649-bd5a-4614367edf44 is now active with session: 8b390bf9-9a20-4516-b216-337ca4dc91cb","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679889905Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679901458Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604,"session_id":"8b390bf9-9a20-4516-b216-337ca4dc91cb"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679913241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604} {"msg":"Activated sub_volume 63049b96-104d-4649-bd5a-4614367edf44","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:27.679924053Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 102 len 4096 data: 5 02/30 Write block 215 len 20480 data: 5 6 7 8 8 03/30 Write block 22 len 8192 data: 7 6 04/30 Read block 262 len 12288 05/30 Read block 392 len 8192 06/30 Write block 477 len 12288 data: 7 7 7 07/30 Write block 208 len 40960 data: 10 12 11 11 9 6 5 6 7 8 08/30 Flush 09/30 Write block 115 len 24576 data: 13 14 12 14 12 12 10/30 Write block 582 len 4096 data: 7 11/30 Read block 446 len 28672 12/30 Read block 199 len 32768 13/30 Read block 403 len 28672 14/30 Read block 265 len 28672 15/30 Read block 111 len 32768 16/30 Flush 17/30 Write block 201 len 12288 data: 12 9 9 18/30 Read block 303 len 8192 19/30 Write block 184 len 20480 data: 14 13 10 9 9 20/30 Read block 568 len 12288 21/30 Read block 220 len 36864 22/30 Read block 495 len 16384 23/30 Read block 424 len 4096 24/30 Read block 119 len 32768 25/30 Read block 235 len 24576 26/30 Write block 551 len 4096 data: 8 27/30 Write block 206 len 32768 data: 9 8 11 13 12 12 10 7 28/30 Write block 230 len 12288 data: 13 9 9 29/30 Read block 243 len 16384 30/30 Write block 209 len 16384 data: 14 13 13 11 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:27.699159623Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1604} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 156 156 156 F F F 1 020-039 104 104 104 170 170 170 F F F 2 040-059 101 101 101 163 163 163 F F F 3 060-079 100 100 100 162 162 162 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 104 104 104 171 171 171 F F F 6 120-139 104 104 104 171 171 171 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 100 100 100 162 162 162 F F F 9 180-199 95 95 95 156 156 156 T T T 10 200-219 104 104 104 170 170 170 T T T 11 220-239 97 97 97 159 159 159 T T T 12 240-259 101 101 101 163 163 163 F F F 13 260-279 98 98 98 160 160 160 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 102 102 102 168 168 168 F F F 16 320-339 95 95 95 156 156 156 F F F 17 340-359 101 101 101 163 163 163 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 100 100 100 162 162 162 F F F 20 400-419 101 101 101 163 163 163 F F F 21 420-439 102 102 102 165 165 165 F F F 22 440-459 102 102 102 168 168 168 F F F 23 460-479 104 104 104 170 170 170 F F F 24 480-499 98 98 98 160 160 160 F F F 25 500-519 102 102 102 166 166 166 F F F 26 520-539 102 102 102 166 166 166 F F F 27 540-559 102 102 102 168 168 168 T T T 28 560-579 102 102 102 168 168 168 F F F 29 580-599 104 104 104 171 171 171 F F F Max gen: 104, Max flush: 171 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 156 156 156 F F F 1 020-039 104 104 104 170 170 170 F F F 2 040-059 101 101 101 163 163 163 F F F 3 060-079 100 100 100 162 162 162 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 104 104 104 171 171 171 F F F 6 120-139 104 104 104 171 171 171 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 100 100 100 162 162 162 F F F 9 180-199 105 105 105 172 172 172 F F F 10 200-219 105 105 105 172 172 172 F F F 11 220-239 105 105 105 172 172 172 F F F 12 240-259 101 101 101 163 163 163 F F F 13 260-279 98 98 98 160 160 160 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 102 102 102 168 168 168 F F F 16 320-339 95 95 95 156 156 156 F F F 17 340-359 101 101 101 163 163 163 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 100 100 100 162 162 162 F F F 20 400-419 101 101 101 163 163 163 F F F 21 420-439 102 102 102 165 165 165 F F F 22 440-459 102 102 102 168 168 168 F F F 23 460-479 104 104 104 170 170 170 F F F 24 480-499 98 98 98 160 160 160 F F F 25 500-519 102 102 102 166 166 166 F F F 26 520-539 102 102 102 166 166 166 F F F 27 540-559 105 105 105 172 172 172 F F F 28 560-579 102 102 102 168 168 168 F F F 29 580-599 104 104 104 171 171 171 F F F Max gen: 105, Max flush: 172 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:31.961571253Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.006805763Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.007560616Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.007611988Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.007630501Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.007649545Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612} {"msg":"Crucible d8760582-7d78-4ec5-bca7-0ef6d9151a80 has session id: a62172d2-056b-493c-a79e-468cdd244cc4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.008243704Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Upstairs opts: Upstairs UUID: d8760582-7d78-4ec5-bca7-0ef6d9151a80, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.008290414Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Crucible stats registered with UUID: d8760582-7d78-4ec5-bca7-0ef6d9151a80","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.008315689Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:32.008336774Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.00882838Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.008852925Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612} {"msg":"d8760582-7d78-4ec5-bca7-0ef6d9151a80 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.008926951Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.008945015Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"0","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.008980643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"0","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.009006758Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"1","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.009033774Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"1","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.009057679Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"2","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.009081154Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"2","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.009174795Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"io task","client":"0","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.009215523Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"io task","client":"2","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.00924566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"io task","client":"1","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:50537)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.009275196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"io task","client":"0","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:64699)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.00938355Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"io task","client":"1","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:32887)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.009449054Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"io task","client":"2","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.0098936Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"0","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.010072748Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"2","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.010205547Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"1","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.010631479Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"2","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.010659815Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"1","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.010680139Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"0","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011205563Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011231649Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"[0]R flush_numbers[0..12]: [156, 170, 163, 162, 133, 171, 171, 169, 162, 172, 172, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011256884Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"[0]R generation[0..12]: [95, 104, 101, 100, 81, 104, 104, 103, 100, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011272127Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.01128662Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"[1]R flush_numbers[0..12]: [156, 170, 163, 162, 133, 171, 171, 169, 162, 172, 172, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011304204Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"[1]R generation[0..12]: [95, 104, 101, 100, 81, 104, 104, 103, 100, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011318167Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.01133186Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"[2]R flush_numbers[0..12]: [156, 170, 163, 162, 133, 171, 171, 169, 162, 172, 172, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011345313Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"[2]R generation[0..12]: [95, 104, 101, 100, 81, 104, 104, 103, 100, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011358696Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011372189Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011386222Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011409687Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.0114244Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011437253Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"":"downstairs","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011450146Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011468329Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"0","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011481422Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"1","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011494215Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"client":"2","":"downstairs","session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011509428Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011521591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"d8760582-7d78-4ec5-bca7-0ef6d9151a80 is now active with session: a62172d2-056b-493c-a79e-468cdd244cc4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011533984Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011545506Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612,"session_id":"a62172d2-056b-493c-a79e-468cdd244cc4"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011557349Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612} {"msg":"Activated sub_volume d8760582-7d78-4ec5-bca7-0ef6d9151a80","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:32.011568141Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 194 len 40960 data: 7 7 6 8 7 7 11 13 10 10 02/30 Flush 03/30 Write block 519 len 36864 data: 11 8 9 10 11 13 12 13 13 04/30 Flush 05/30 Write block 348 len 24576 data: 6 6 7 7 5 6 06/30 Write block 400 len 12288 data: 9 8 6 07/30 Read block 488 len 12288 08/30 Write block 14 len 24576 data: 6 5 6 5 5 5 09/30 Read block 91 len 16384 10/30 Flush 11/30 Write block 108 len 16384 data: 6 5 4 5 12/30 Read block 53 len 4096 13/30 Read block 455 len 16384 14/30 Read block 338 len 16384 15/30 Flush 16/30 Write block 40 len 36864 data: 10 9 9 12 15 11 12 12 14 17/30 Write block 173 len 8192 data: 7 6 18/30 Read block 332 len 40960 19/30 Read block 33 len 32768 20/30 Read block 347 len 12288 21/30 Read block 297 len 12288 22/30 Write block 456 len 4096 data: 7 23/30 Read block 71 len 40960 24/30 Flush 25/30 Flush 26/30 Write block 456 len 36864 data: 8 9 9 8 8 8 9 10 10 27/30 Read block 239 len 20480 28/30 Read block 341 len 16384 29/30 Write block 206 len 4096 data: 10 30/30 Write block 194 len 36864 data: 8 8 7 9 8 8 12 14 11 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:32.032130318Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1612} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 175 175 175 F F F 1 020-039 104 104 104 170 170 170 F F F 2 040-059 106 106 106 177 177 177 F F F 3 060-079 100 100 100 162 162 162 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 106 106 106 176 176 176 F F F 6 120-139 104 104 104 171 171 171 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 106 106 106 177 177 177 F F F 9 180-199 106 106 106 173 173 173 T T T 10 200-219 106 106 106 173 173 173 T T T 11 220-239 105 105 105 172 172 172 F F F 12 240-259 101 101 101 163 163 163 F F F 13 260-279 98 98 98 160 160 160 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 102 102 102 168 168 168 F F F 16 320-339 95 95 95 156 156 156 F F F 17 340-359 106 106 106 175 175 175 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 100 100 100 162 162 162 F F F 20 400-419 106 106 106 175 175 175 F F F 21 420-439 102 102 102 165 165 165 F F F 22 440-459 106 106 106 177 177 177 T T T 23 460-479 104 104 104 170 170 170 T T T 24 480-499 98 98 98 160 160 160 F F F 25 500-519 106 106 106 174 174 174 F F F 26 520-539 106 106 106 174 174 174 F F F 27 540-559 105 105 105 172 172 172 F F F 28 560-579 102 102 102 168 168 168 F F F 29 580-599 104 104 104 171 171 171 F F F Max gen: 106, Max flush: 177 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 175 175 175 F F F 1 020-039 104 104 104 170 170 170 F F F 2 040-059 106 106 106 177 177 177 F F F 3 060-079 100 100 100 162 162 162 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 106 106 106 176 176 176 F F F 6 120-139 104 104 104 171 171 171 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 106 106 106 177 177 177 F F F 9 180-199 107 107 107 178 178 178 F F F 10 200-219 107 107 107 178 178 178 F F F 11 220-239 105 105 105 172 172 172 F F F 12 240-259 101 101 101 163 163 163 F F F 13 260-279 98 98 98 160 160 160 F F F 14 280-299 92 92 92 151 151 151 F F F 15 300-319 102 102 102 168 168 168 F F F 16 320-339 95 95 95 156 156 156 F F F 17 340-359 106 106 106 175 175 175 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 100 100 100 162 162 162 F F F 20 400-419 106 106 106 175 175 175 F F F 21 420-439 102 102 102 165 165 165 F F F 22 440-459 107 107 107 178 178 178 F F F 23 460-479 107 107 107 178 178 178 F F F 24 480-499 98 98 98 160 160 160 F F F 25 500-519 106 106 106 174 174 174 F F F 26 520-539 106 106 106 174 174 174 F F F 27 540-559 105 105 105 172 172 172 F F F 28 560-579 102 102 102 168 168 168 F F F 29 580-599 104 104 104 171 171 171 F F F Max gen: 107, Max flush: 178 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.265864371Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.311865187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.312546684Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.312590543Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.312605496Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.312620149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620} {"msg":"Crucible 290f7242-871c-4c09-ae87-a8b60e2ca5d4 has session id: 1e632c87-6931-4d06-b3db-1e6ce38c6b30","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313164407Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"Upstairs opts: Upstairs UUID: 290f7242-871c-4c09-ae87-a8b60e2ca5d4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313199194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"Crucible stats registered with UUID: 290f7242-871c-4c09-ae87-a8b60e2ca5d4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313214388Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:36.313227151Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313626117Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313658613Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620} {"msg":"290f7242-871c-4c09-ae87-a8b60e2ca5d4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313713985Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313740411Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"0","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313768167Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"0","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.31378334Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"1","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313801224Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"1","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313815518Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"2","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.31382979Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"2","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313842883Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"io task","client":"2","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313856076Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"io task","client":"0","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.313869109Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"io task","client":"1","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:36.314005308Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"io task","client":"1","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:36.314046277Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"io task","client":"1","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"ds_connection connected from Ok(127.0.0.1:63148)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.314074333Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"io task","client":"0","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"ds_connection connected from Ok(127.0.0.1:45053)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.314100139Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"io task","client":"2","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:36.314124344Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.314209643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"io task","client":"1","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.314684215Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"2","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.314712341Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"0","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.315329464Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"2","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.315451Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"0","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.315892886Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:36.315987756Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.315554714Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"io task","client":"1","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"ds_connection connected from Ok(127.0.0.1:59325)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.315763669Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"io task","client":"1","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.316746251Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"1","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317236646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"1","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"[0]R flush_numbers[0..12]: [175, 170, 177, 162, 133, 176, 171, 169, 177, 178, 178, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317801928Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"[0]R generation[0..12]: [106, 104, 106, 100, 81, 106, 104, 103, 106, 107, 107, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317827024Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317845208Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"[1]R flush_numbers[0..12]: [175, 170, 177, 162, 133, 176, 171, 169, 177, 178, 178, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317862161Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"[1]R generation[0..12]: [106, 104, 106, 100, 81, 106, 104, 103, 106, 107, 107, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317878625Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317895439Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"[2]R flush_numbers[0..12]: [175, 170, 177, 162, 133, 176, 171, 169, 177, 178, 178, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317918504Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"[2]R generation[0..12]: [106, 104, 106, 100, 81, 106, 104, 103, 106, 107, 107, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317935847Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317952571Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317967984Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317983948Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.317999491Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.318015125Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"":"downstairs","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.318031118Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.318046951Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"0","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.318063305Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"1","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.31808662Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"client":"2","":"downstairs","session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.318103013Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.318117917Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"290f7242-871c-4c09-ae87-a8b60e2ca5d4 is now active with session: 1e632c87-6931-4d06-b3db-1e6ce38c6b30","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.31813339Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.318149223Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620,"session_id":"1e632c87-6931-4d06-b3db-1e6ce38c6b30"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.318164187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620} {"msg":"Activated sub_volume 290f7242-871c-4c09-ae87-a8b60e2ca5d4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:46.31817909Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 167 len 20480 data: 10 9 12 9 8 03/30 Write block 592 len 32768 data: 6 6 5 3 4 4 4 3 04/30 Write block 228 len 24576 data: 11 10 14 10 10 10 05/30 Write block 302 len 32768 data: 12 12 12 10 10 9 10 9 06/30 Read block 386 len 40960 07/30 Write block 246 len 12288 data: 8 8 8 08/30 Read block 315 len 36864 09/30 Read block 554 len 32768 10/30 Read block 52 len 12288 11/30 Read block 12 len 20480 12/30 Write block 480 len 28672 data: 7 7 8 10 9 10 10 13/30 Write block 280 len 28672 data: 8 9 9 9 10 11 10 14/30 Write block 61 len 12288 data: 7 8 8 15/30 Read block 489 len 12288 16/30 Flush 17/30 Write block 330 len 28672 data: 7 11 12 13 15 16 13 18/30 Flush 19/30 Write block 355 len 20480 data: 6 6 7 9 9 20/30 Read block 177 len 20480 21/30 Read block 284 len 16384 22/30 Read block 491 len 28672 23/30 Read block 199 len 24576 24/30 Read block 460 len 20480 25/30 Read block 275 len 36864 26/30 Read block 65 len 4096 27/30 Write block 402 len 28672 data: 7 8 8 9 10 12 10 28/30 Write block 385 len 24576 data: 8 9 9 9 9 9 29/30 Read block 2 len 24576 30/30 Write block 48 len 12288 data: 15 15 13 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 5 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:46.33949806Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1620} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 175 175 175 F F F 1 020-039 104 104 104 170 170 170 F F F 2 040-059 106 106 106 177 177 177 T T T 3 060-079 108 108 108 180 180 180 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 106 106 106 176 176 176 F F F 6 120-139 104 104 104 171 171 171 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 108 108 108 180 180 180 F F F 9 180-199 107 107 107 178 178 178 F F F 10 200-219 107 107 107 178 178 178 F F F 11 220-239 108 108 108 180 180 180 F F F 12 240-259 108 108 108 180 180 180 F F F 13 260-279 98 98 98 160 160 160 F F F 14 280-299 108 108 108 180 180 180 F F F 15 300-319 108 108 108 180 180 180 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 106 106 106 175 175 175 T T T 18 360-379 98 98 98 160 160 160 F F F 19 380-399 100 100 100 162 162 162 T T T 20 400-419 106 106 106 175 175 175 T T T 21 420-439 102 102 102 165 165 165 F F F 22 440-459 107 107 107 178 178 178 F F F 23 460-479 107 107 107 178 178 178 F F F 24 480-499 108 108 108 180 180 180 F F F 25 500-519 106 106 106 174 174 174 F F F 26 520-539 106 106 106 174 174 174 F F F 27 540-559 105 105 105 172 172 172 F F F 28 560-579 102 102 102 168 168 168 F F F 29 580-599 108 108 108 180 180 180 F F F Max gen: 108, Max flush: 181 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 175 175 175 F F F 1 020-039 104 104 104 170 170 170 F F F 2 040-059 109 109 109 182 182 182 F F F 3 060-079 108 108 108 180 180 180 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 106 106 106 176 176 176 F F F 6 120-139 104 104 104 171 171 171 F F F 7 140-159 103 103 103 169 169 169 F F F 8 160-179 108 108 108 180 180 180 F F F 9 180-199 107 107 107 178 178 178 F F F 10 200-219 107 107 107 178 178 178 F F F 11 220-239 108 108 108 180 180 180 F F F 12 240-259 108 108 108 180 180 180 F F F 13 260-279 98 98 98 160 160 160 F F F 14 280-299 108 108 108 180 180 180 F F F 15 300-319 108 108 108 180 180 180 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 109 109 109 182 182 182 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 109 109 109 182 182 182 F F F 20 400-419 109 109 109 182 182 182 F F F 21 420-439 102 102 102 165 165 165 F F F 22 440-459 107 107 107 178 178 178 F F F 23 460-479 107 107 107 178 178 178 F F F 24 480-499 108 108 108 180 180 180 F F F 25 500-519 106 106 106 174 174 174 F F F 26 520-539 106 106 106 174 174 174 F F F 27 540-559 105 105 105 172 172 172 F F F 28 560-579 102 102 102 168 168 168 F F F 29 580-599 108 108 108 180 180 180 F F F Max gen: 109, Max flush: 182 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.572516079Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.617415747Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.618155426Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.618202266Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.618225381Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.618243225Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628} {"msg":"Crucible 61279c23-1f79-4e74-80b9-c82e4e0c055b has session id: 03a01ac1-f92a-4974-b9c1-773097ece440","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.618862319Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Upstairs opts: Upstairs UUID: 61279c23-1f79-4e74-80b9-c82e4e0c055b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.618904718Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Crucible stats registered with UUID: 61279c23-1f79-4e74-80b9-c82e4e0c055b","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.618928233Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:50.618948617Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.619395534Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.619420469Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628} {"msg":"61279c23-1f79-4e74-80b9-c82e4e0c055b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.619499606Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.61951747Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"0","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.619545996Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"0","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.619559709Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"1","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.619573632Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"1","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.619586335Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"2","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.619599968Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"2","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.619616471Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"io task","client":"2","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.619630514Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"io task","client":"0","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.619644628Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"io task","client":"1","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:50.619737538Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"io task","client":"2","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:50.619812674Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"io task","client":"2","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"ds_connection connected from Ok(127.0.0.1:64072)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.61984136Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"io task","client":"0","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"ds_connection connected from Ok(127.0.0.1:55185)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.619968578Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"io task","client":"1","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:25:50.619998524Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.620019329Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"io task","client":"2","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.620471476Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"1","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.620601864Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"0","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.621062394Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"1","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.621206345Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"0","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.621615373Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:25:50.621707393Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.620838526Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"io task","client":"2","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"ds_connection connected from Ok(127.0.0.1:43132)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.620976946Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"io task","client":"2","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.622034264Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"2","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.622552045Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"2","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"[0]R flush_numbers[0..12]: [175, 170, 182, 180, 133, 176, 171, 169, 180, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623134891Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"[0]R generation[0..12]: [106, 104, 109, 108, 81, 106, 104, 103, 108, 107, 107, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623169089Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623188153Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"[1]R flush_numbers[0..12]: [175, 170, 182, 180, 133, 176, 171, 169, 180, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623205757Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"[1]R generation[0..12]: [106, 104, 109, 108, 81, 106, 104, 103, 108, 107, 107, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.62322287Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623240384Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"[2]R flush_numbers[0..12]: [175, 170, 182, 180, 133, 176, 171, 169, 180, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623256948Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"[2]R generation[0..12]: [106, 104, 109, 108, 81, 106, 104, 103, 108, 107, 107, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623273461Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623295196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:26:00.623311459Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623331334Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623347297Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623363211Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"":"downstairs","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623379464Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623393847Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"0","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623409781Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"1","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623426354Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"client":"2","":"downstairs","session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623439657Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.62345156Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"61279c23-1f79-4e74-80b9-c82e4e0c055b is now active with session: 03a01ac1-f92a-4974-b9c1-773097ece440","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-17T22:26:00.623464002Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623478225Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628,"session_id":"03a01ac1-f92a-4974-b9c1-773097ece440"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.623490328Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628} {"msg":"Activated sub_volume 61279c23-1f79-4e74-80b9-c82e4e0c055b","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:00.62350131Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 31 len 16384 data: 6 5 6 6 02/30 Write block 69 len 12288 data: 11 13 12 03/30 Read block 265 len 24576 04/30 Read block 373 len 28672 05/30 Read block 184 len 8192 06/30 Flush 07/30 Write block 153 len 36864 data: 12 14 12 12 11 11 12 11 10 08/30 Read block 51 len 20480 09/30 Read block 94 len 28672 10/30 Read block 312 len 32768 11/30 Read block 245 len 8192 12/30 Read block 219 len 4096 13/30 Read block 145 len 36864 14/30 Write block 284 len 40960 data: 11 12 11 7 8 6 9 6 5 5 15/30 Write block 28 len 40960 data: 8 8 6 7 6 7 7 9 8 10 16/30 Write block 108 len 4096 data: 7 17/30 Read block 163 len 12288 18/30 Write block 269 len 32768 data: 7 8 8 9 9 8 7 8 19/30 Read block 108 len 36864 20/30 Read block 243 len 8192 21/30 Write block 426 len 40960 data: 8 9 11 10 11 10 10 10 9 7 22/30 Read block 12 len 20480 23/30 Write block 540 len 24576 data: 11 10 12 11 13 12 24/30 Read block 124 len 8192 25/30 Flush 26/30 Write block 246 len 8192 data: 9 9 27/30 Write block 278 len 24576 data: 7 6 9 10 10 10 28/30 Read block 22 len 36864 29/30 Write block 501 len 40960 data: 13 11 11 8 7 6 6 7 9 10 30/30 Write block 114 len 8192 data: 11 14 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:26:00.644793204Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1628} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 175 175 175 F F F 1 020-039 110 110 110 184 184 184 F F F 2 040-059 109 109 109 182 182 182 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 110 110 110 184 184 184 T T T 6 120-139 104 104 104 171 171 171 F F F 7 140-159 110 110 110 184 184 184 F F F 8 160-179 110 110 110 184 184 184 F F F 9 180-199 107 107 107 178 178 178 F F F 10 200-219 107 107 107 178 178 178 F F F 11 220-239 108 108 108 180 180 180 F F F 12 240-259 108 108 108 180 180 180 T T T 13 260-279 110 110 110 184 184 184 T T T 14 280-299 110 110 110 184 184 184 T T T 15 300-319 108 108 108 180 180 180 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 109 109 109 182 182 182 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 109 109 109 182 182 182 F F F 20 400-419 109 109 109 182 182 182 F F F 21 420-439 110 110 110 184 184 184 F F F 22 440-459 107 107 107 178 178 178 F F F 23 460-479 107 107 107 178 178 178 F F F 24 480-499 108 108 108 180 180 180 F F F 25 500-519 106 106 106 174 174 174 T T T 26 520-539 106 106 106 174 174 174 F F F 27 540-559 110 110 110 184 184 184 F F F 28 560-579 102 102 102 168 168 168 F F F 29 580-599 108 108 108 180 180 180 F F F Max gen: 110, Max flush: 184 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 175 175 175 F F F 1 020-039 110 110 110 184 184 184 F F F 2 040-059 109 109 109 182 182 182 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 111 111 111 185 185 185 F F F 6 120-139 104 104 104 171 171 171 F F F 7 140-159 110 110 110 184 184 184 F F F 8 160-179 110 110 110 184 184 184 F F F 9 180-199 107 107 107 178 178 178 F F F 10 200-219 107 107 107 178 178 178 F F F 11 220-239 108 108 108 180 180 180 F F F 12 240-259 111 111 111 185 185 185 F F F 13 260-279 111 111 111 185 185 185 F F F 14 280-299 111 111 111 185 185 185 F F F 15 300-319 108 108 108 180 180 180 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 109 109 109 182 182 182 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 109 109 109 182 182 182 F F F 20 400-419 109 109 109 182 182 182 F F F 21 420-439 110 110 110 184 184 184 F F F 22 440-459 107 107 107 178 178 178 F F F 23 460-479 107 107 107 178 178 178 F F F 24 480-499 108 108 108 180 180 180 F F F 25 500-519 111 111 111 185 185 185 F F F 26 520-539 106 106 106 174 174 174 F F F 27 540-559 110 110 110 184 184 184 F F F 28 560-579 102 102 102 168 168 168 F F F 29 580-599 108 108 108 180 180 180 F F F Max gen: 111, Max flush: 185 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.888945067Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.933043793Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.933692782Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.933734002Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.933749535Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.933764438Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636} {"msg":"Crucible c50793ec-5e6c-4bcf-92ac-342d68185750 has session id: d8f7281c-23aa-49ac-9437-f54e858321b5","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.934260305Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Upstairs opts: Upstairs UUID: c50793ec-5e6c-4bcf-92ac-342d68185750, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.934345333Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Crucible stats registered with UUID: c50793ec-5e6c-4bcf-92ac-342d68185750","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.934366288Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:26:14.934379231Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.934826737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.934849572Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636} {"msg":"c50793ec-5e6c-4bcf-92ac-342d68185750 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.934918477Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.934936221Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"0","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.934963807Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"0","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.93497786Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"1","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.935006346Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"1","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.935022059Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"2","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.935036743Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"2","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.935059017Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"io task","client":"2","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.935082913Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"io task","client":"0","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.935098616Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"io task","client":"1","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:51770)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.9351194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"io task","client":"0","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:26:14.935148367Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"io task","client":"2","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:26:14.935170961Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"io task","client":"2","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:43866)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.935233715Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"io task","client":"1","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:26:14.935271043Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.935453582Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"io task","client":"2","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.93585885Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"0","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.935897658Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"1","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.936513981Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"0","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.936550079Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"1","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.937119642Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:14.937146118Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.936845633Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"io task","client":"2","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58362)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.93697261Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"io task","client":"2","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.938133221Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"2","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.938725369Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"2","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"[0]R flush_numbers[0..12]: [175, 184, 182, 183, 133, 185, 171, 184, 184, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939316526Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"[0]R generation[0..12]: [106, 110, 109, 110, 81, 111, 104, 110, 110, 107, 107, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939346693Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939365547Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"[1]R flush_numbers[0..12]: [175, 184, 182, 183, 133, 185, 171, 184, 184, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939382811Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"[1]R generation[0..12]: [106, 110, 109, 110, 81, 111, 104, 110, 110, 107, 107, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939399514Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939416588Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"[2]R flush_numbers[0..12]: [175, 184, 182, 183, 133, 185, 171, 184, 184, 178, 178, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939433421Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"[2]R generation[0..12]: [106, 110, 109, 110, 81, 111, 104, 110, 110, 107, 107, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939450495Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939467949Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939483723Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939499836Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939515569Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939536404Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"":"downstairs","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939553087Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.93956753Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"0","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939583874Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"1","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939599968Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"client":"2","":"downstairs","session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:26:24.939615941Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939638516Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"c50793ec-5e6c-4bcf-92ac-342d68185750 is now active with session: d8f7281c-23aa-49ac-9437-f54e858321b5","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939653909Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939668442Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636,"session_id":"d8f7281c-23aa-49ac-9437-f54e858321b5"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939682965Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636} {"msg":"Activated sub_volume c50793ec-5e6c-4bcf-92ac-342d68185750","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:24.939696558Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 214 len 20480 data: 6 7 8 9 9 02/30 Write block 135 len 28672 data: 6 6 5 7 9 8 7 03/30 Write block 205 len 4096 data: 11 04/30 Read block 136 len 4096 05/30 Write block 272 len 24576 data: 10 10 9 8 9 6 06/30 Read block 98 len 40960 07/30 Read block 394 len 24576 08/30 Read block 281 len 4096 09/30 Read block 581 len 8192 10/30 Read block 528 len 28672 11/30 Read block 381 len 24576 12/30 Write block 166 len 24576 data: 15 11 10 13 10 9 13/30 Read block 321 len 40960 14/30 Read block 538 len 32768 15/30 Read block 380 len 24576 16/30 Read block 525 len 20480 17/30 Write block 555 len 36864 data: 10 10 10 11 10 12 11 13 12 18/30 Write block 538 len 36864 data: 7 10 12 11 13 12 14 13 12 19/30 Write block 402 len 24576 data: 8 9 9 10 11 13 20/30 Write block 422 len 12288 data: 4 6 7 21/30 Read block 579 len 8192 22/30 Read block 259 len 4096 23/30 Read block 336 len 12288 24/30 Read block 155 len 20480 25/30 Write block 191 len 8192 data: 5 5 26/30 Write block 287 len 4096 data: 8 27/30 Write block 2 len 20480 data: 6 6 8 7 9 28/30 Read block 348 len 24576 29/30 Write block 157 len 4096 data: 12 30/30 Read block 432 len 36864 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 30 29 28 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:26:24.959386727Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1636} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 175 175 175 T T T 1 020-039 110 110 110 184 184 184 F F F 2 040-059 109 109 109 182 182 182 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 111 111 111 185 185 185 F F F 6 120-139 104 104 104 171 171 171 T T T 7 140-159 110 110 110 184 184 184 T T T 8 160-179 110 110 110 184 184 184 T T T 9 180-199 107 107 107 178 178 178 T T T 10 200-219 107 107 107 178 178 178 T T T 11 220-239 108 108 108 180 180 180 F F F 12 240-259 111 111 111 185 185 185 F F F 13 260-279 111 111 111 185 185 185 T T T 14 280-299 111 111 111 185 185 185 T T T 15 300-319 108 108 108 180 180 180 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 109 109 109 182 182 182 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 109 109 109 182 182 182 F F F 20 400-419 109 109 109 182 182 182 T T T 21 420-439 110 110 110 184 184 184 T T T 22 440-459 107 107 107 178 178 178 F F F 23 460-479 107 107 107 178 178 178 F F F 24 480-499 108 108 108 180 180 180 F F F 25 500-519 111 111 111 185 185 185 F F F 26 520-539 106 106 106 174 174 174 T T T 27 540-559 110 110 110 184 184 184 T T T 28 560-579 102 102 102 168 168 168 T T T 29 580-599 108 108 108 180 180 180 F F F Max gen: 111, Max flush: 185 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 186 186 186 F F F 1 020-039 110 110 110 184 184 184 F F F 2 040-059 109 109 109 182 182 182 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 81 81 81 133 133 133 F F F 5 100-119 111 111 111 185 185 185 F F F 6 120-139 112 112 112 186 186 186 F F F 7 140-159 112 112 112 186 186 186 F F F 8 160-179 112 112 112 186 186 186 F F F 9 180-199 112 112 112 186 186 186 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 108 108 108 180 180 180 F F F 12 240-259 111 111 111 185 185 185 F F F 13 260-279 112 112 112 186 186 186 F F F 14 280-299 112 112 112 186 186 186 F F F 15 300-319 108 108 108 180 180 180 F F F 16 320-339 108 108 108 181 181 181 F F F 17 340-359 109 109 109 182 182 182 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 109 109 109 182 182 182 F F F 20 400-419 112 112 112 186 186 186 F F F 21 420-439 112 112 112 186 186 186 F F F 22 440-459 107 107 107 178 178 178 F F F 23 460-479 107 107 107 178 178 178 F F F 24 480-499 108 108 108 180 180 180 F F F 25 500-519 111 111 111 185 185 185 F F F 26 520-539 112 112 112 186 186 186 F F F 27 540-559 112 112 112 186 186 186 F F F 28 560-579 112 112 112 186 186 186 F F F 29 580-599 108 108 108 180 180 180 F F F Max gen: 112, Max flush: 186 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.243888664Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.290014427Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.290739643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.290791934Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.290810889Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.290828672Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644} {"msg":"Crucible f3f39c88-0dd8-4389-877c-b91f3a2e894e has session id: a1a28254-58b8-45c6-a2b7-525513ad89f3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.291352245Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Upstairs opts: Upstairs UUID: f3f39c88-0dd8-4389-877c-b91f3a2e894e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.291399196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Crucible stats registered with UUID: f3f39c88-0dd8-4389-877c-b91f3a2e894e","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.291417449Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:26:39.291430032Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.291910866Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.291934711Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644} {"msg":"f3f39c88-0dd8-4389-877c-b91f3a2e894e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292012868Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292031302Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"0","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292058718Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"0","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292072531Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"1","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292086774Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"1","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292099887Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"2","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292113459Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"2","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292126282Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"io task","client":"2","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292139755Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"io task","client":"0","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292157529Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"io task","client":"1","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:26:39.292277835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"io task","client":"2","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:58998)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292313763Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"io task","client":"0","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45489)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292339698Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"io task","client":"1","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:26:39.292369875Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"io task","client":"2","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:26:39.292402582Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.292571729Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"io task","client":"2","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.293030917Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"0","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.293056853Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"1","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.293559102Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"1","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.293668255Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"0","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.294147309Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:39.294263454Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.293377483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"io task","client":"2","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:55413)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.293516733Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"io task","client":"2","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.295021637Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"2","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.295559013Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"2","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"[0]R flush_numbers[0..12]: [186, 184, 182, 183, 133, 185, 186, 186, 186, 186, 186, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296134758Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"[0]R generation[0..12]: [112, 110, 109, 110, 81, 111, 112, 112, 112, 112, 112, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296165334Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296187619Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"[1]R flush_numbers[0..12]: [186, 184, 182, 183, 133, 185, 186, 186, 186, 186, 186, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296215545Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"[1]R generation[0..12]: [112, 110, 109, 110, 81, 111, 112, 112, 112, 112, 112, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296244371Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296275558Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"[2]R flush_numbers[0..12]: [186, 184, 182, 183, 133, 185, 186, 186, 186, 186, 186, 180]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296308155Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"[2]R generation[0..12]: [112, 110, 109, 110, 81, 111, 112, 112, 112, 112, 112, 108]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296336591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296363687Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Max found gen is 113","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296392483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Generation requested: 114 >= found:113","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296419139Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296446215Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296460558Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"":"downstairs","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296482703Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"Wait for a query_work_queue command to finish before sending IO a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296510379Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"0","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296535344Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"1","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296555338Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"client":"2","":"downstairs","session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296570212Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296588345Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"f3f39c88-0dd8-4389-877c-b91f3a2e894e is now active with session: a1a28254-58b8-45c6-a2b7-525513ad89f3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.29661126Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296632485Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644,"session_id":"a1a28254-58b8-45c6-a2b7-525513ad89f3"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296648588Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644} {"msg":"Activated sub_volume f3f39c88-0dd8-4389-877c-b91f3a2e894e","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:26:49.296668113Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 390 len 36864 data: 10 5 5 9 9 9 10 10 10 04/30 Write block 291 len 12288 data: 7 6 6 05/30 Read block 10 len 12288 06/30 Write block 0 len 36864 data: 4 5 7 7 9 8 10 12 10 07/30 Read block 28 len 4096 08/30 Read block 15 len 32768 09/30 Write block 235 len 4096 data: 11 10/30 Read block 118 len 20480 11/30 Write block 585 len 40960 data: 11 8 8 8 10 8 9 7 7 6 12/30 Read block 185 len 12288 13/30 Write block 329 len 16384 data: 7 8 12 13 14/30 Read block 59 len 36864 15/30 Write block 240 len 20480 data: 7 6 6 6 6 16/30 Read block 225 len 40960 17/30 Write block 417 len 4096 data: 6 18/30 Write block 97 len 40960 data: 6 6 6 4 5 6 5 6 6 6 19/30 Read block 195 len 28672 20/30 Read block 498 len 28672 21/30 Write block 343 len 12288 data: 6 7 6 22/30 Read block 336 len 20480 23/30 Write block 292 len 36864 data: 7 7 7 7 10 11 11 8 10 24/30 Write block 585 len 36864 data: 12 9 9 9 11 9 10 8 8 25/30 Write block 475 len 4096 data: 11 26/30 Write block 146 len 24576 data: 9 10 11 11 9 12 27/30 Read block 499 len 16384 28/30 Read block 291 len 20480 29/30 Write block 560 len 28672 data: 13 12 14 13 11 10 9 30/30 Read block 55 len 8192 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 9 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 7 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 28 26 26 80 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:26:49.317378401Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1644} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 186 186 186 T T T 1 020-039 110 110 110 184 184 184 F F F 2 040-059 109 109 109 182 182 182 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 81 81 81 133 133 133 T T T 5 100-119 111 111 111 185 185 185 T T T 6 120-139 112 112 112 186 186 186 F F F 7 140-159 112 112 112 186 186 186 T T T 8 160-179 112 112 112 186 186 186 F F F 9 180-199 112 112 112 186 186 186 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 108 108 108 180 180 180 T T T 12 240-259 111 111 111 185 185 185 T T T 13 260-279 112 112 112 186 186 186 F F F 14 280-299 112 112 112 186 186 186 T T T 15 300-319 108 108 108 180 180 180 T T T 16 320-339 108 108 108 181 181 181 T T T 17 340-359 109 109 109 182 182 182 T T T 18 360-379 98 98 98 160 160 160 F F F 19 380-399 109 109 109 182 182 182 T T T 20 400-419 112 112 112 186 186 186 T T T 21 420-439 112 112 112 186 186 186 F F F 22 440-459 107 107 107 178 178 178 F F F 23 460-479 107 107 107 178 178 178 T T T 24 480-499 108 108 108 180 180 180 F F F 25 500-519 111 111 111 185 185 185 F F F 26 520-539 112 112 112 186 186 186 F F F 27 540-559 112 112 112 186 186 186 F F F 28 560-579 112 112 112 186 186 186 T T T 29 580-599 108 108 108 180 180 180 T T T Max gen: 112, Max flush: 186 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 187 187 187 F F F 1 020-039 110 110 110 184 184 184 F F F 2 040-059 109 109 109 182 182 182 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 113 113 113 187 187 187 F F F 5 100-119 113 113 113 187 187 187 F F F 6 120-139 112 112 112 186 186 186 F F F 7 140-159 113 113 113 187 187 187 F F F 8 160-179 112 112 112 186 186 186 F F F 9 180-199 112 112 112 186 186 186 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 113 113 113 187 187 187 F F F 12 240-259 113 113 113 187 187 187 F F F 13 260-279 112 112 112 186 186 186 F F F 14 280-299 113 113 113 187 187 187 F F F 15 300-319 113 113 113 187 187 187 F F F 16 320-339 113 113 113 187 187 187 F F F 17 340-359 113 113 113 187 187 187 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 113 113 113 187 187 187 F F F 20 400-419 113 113 113 187 187 187 F F F 21 420-439 112 112 112 186 186 186 F F F 22 440-459 107 107 107 178 178 178 F F F 23 460-479 113 113 113 187 187 187 F F F 24 480-499 108 108 108 180 180 180 F F F 25 500-519 111 111 111 185 185 185 F F F 26 520-539 112 112 112 186 186 186 F F F 27 540-559 112 112 112 186 186 186 F F F 28 560-579 113 113 113 187 187 187 F F F 29 580-599 113 113 113 187 187 187 F F F Max gen: 113, Max flush: 187 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.61151277Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:03.654352753Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.654727074Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.6857608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.686373423Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.68640807Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.686426094Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.686440877Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652} {"msg":"Crucible 4b3c1e13-d542-4dbc-88ec-fba9b11d3563 has session id: 1e46264f-194e-4f29-a7cc-4da8ef66f5bd","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.686980323Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Upstairs opts: Upstairs UUID: 4b3c1e13-d542-4dbc-88ec-fba9b11d3563, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687052199Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Crucible stats registered with UUID: 4b3c1e13-d542-4dbc-88ec-fba9b11d3563","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687074694Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:03.687088257Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687492624Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687517599Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652} {"msg":"4b3c1e13-d542-4dbc-88ec-fba9b11d3563 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687583253Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687601077Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"0","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687631464Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"0","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687646667Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"1","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.68766096Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"1","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687674063Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"2","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687693247Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"2","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.68770716Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"io task","client":"2","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687726975Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"io task","client":"0","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687749869Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"io task","client":"1","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:39081)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687932109Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"io task","client":"0","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:46495)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.687979829Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"io task","client":"2","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:37341)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.688016527Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"io task","client":"1","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.68877319Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"1","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.688804587Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"2","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.689022044Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"0","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.689356446Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"2","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.689384752Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"1","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.689516981Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"0","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690028931Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690054187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"[0]R flush_numbers[0..12]: [187, 184, 182, 183, 187, 187, 186, 187, 186, 186, 186, 187]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690071641Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"[0]R generation[0..12]: [113, 110, 109, 110, 113, 113, 112, 113, 112, 112, 112, 113]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690086774Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690101897Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"[1]R flush_numbers[0..12]: [187, 184, 182, 183, 187, 187, 186, 187, 186, 186, 186, 187]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.69011576Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"[1]R generation[0..12]: [113, 110, 109, 110, 113, 113, 112, 113, 112, 112, 112, 113]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690129133Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690142536Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"[2]R flush_numbers[0..12]: [187, 184, 182, 183, 187, 187, 186, 187, 186, 186, 186, 187]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690156149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"[2]R generation[0..12]: [113, 110, 109, 110, 113, 113, 112, 113, 112, 112, 112, 113]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690169322Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690182985Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690195797Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Generation requested: 116 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.6902086Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690221223Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690233685Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"":"downstairs","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.69025381Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690267353Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"0","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690280816Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"1","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690293648Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"client":"2","":"downstairs","session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690306661Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690318884Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"4b3c1e13-d542-4dbc-88ec-fba9b11d3563 is now active with session: 1e46264f-194e-4f29-a7cc-4da8ef66f5bd","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690331377Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690343179Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652,"session_id":"1e46264f-194e-4f29-a7cc-4da8ef66f5bd"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690355132Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652} {"msg":"Activated sub_volume 4b3c1e13-d542-4dbc-88ec-fba9b11d3563","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:03.690366104Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 537 len 12288 data: 8 8 11 02/30 Read block 424 len 28672 03/30 Read block 257 len 4096 04/30 Write block 445 len 20480 data: 10 12 10 7 7 05/30 Write block 151 len 4096 data: 13 06/30 Write block 546 len 24576 data: 13 10 9 9 9 9 07/30 Flush 08/30 Write block 176 len 40960 data: 8 10 11 13 14 12 12 12 15 14 09/30 Read block 452 len 20480 10/30 Write block 466 len 12288 data: 7 13 14 11/30 Flush 12/30 Write block 507 len 32768 data: 7 8 10 11 7 8 8 9 13/30 Write block 345 len 4096 data: 7 14/30 Read block 110 len 24576 15/30 Write block 408 len 16384 data: 11 8 7 7 16/30 Write block 533 len 24576 data: 11 10 10 7 9 9 17/30 Read block 264 len 32768 18/30 Read block 211 len 40960 19/30 Write block 448 len 40960 data: 8 8 6 8 10 13 11 9 9 10 20/30 Write block 531 len 8192 data: 10 9 21/30 Read block 314 len 32768 22/30 Read block 361 len 28672 23/30 Read block 144 len 40960 24/30 Flush 25/30 Write block 164 len 12288 data: 14 18 16 26/30 Read block 253 len 36864 27/30 Write block 191 len 16384 data: 6 6 5 9 28/30 Write block 163 len 8192 data: 11 15 29/30 Read block 276 len 36864 30/30 Write block 576 len 16384 data: 8 10 10 10 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:03.710256586Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1652} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 187 187 187 F F F 1 020-039 110 110 110 184 184 184 F F F 2 040-059 109 109 109 182 182 182 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 113 113 113 187 187 187 F F F 5 100-119 113 113 113 187 187 187 F F F 6 120-139 112 112 112 186 186 186 F F F 7 140-159 116 116 116 188 188 188 F F F 8 160-179 116 116 116 189 189 189 T T T 9 180-199 116 116 116 189 189 189 T T T 10 200-219 112 112 112 186 186 186 F F F 11 220-239 113 113 113 187 187 187 F F F 12 240-259 113 113 113 187 187 187 F F F 13 260-279 112 112 112 186 186 186 F F F 14 280-299 113 113 113 187 187 187 F F F 15 300-319 113 113 113 187 187 187 F F F 16 320-339 113 113 113 187 187 187 F F F 17 340-359 116 116 116 190 190 190 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 113 113 113 187 187 187 F F F 20 400-419 116 116 116 190 190 190 F F F 21 420-439 112 112 112 186 186 186 F F F 22 440-459 116 116 116 190 190 190 F F F 23 460-479 116 116 116 189 189 189 F F F 24 480-499 108 108 108 180 180 180 F F F 25 500-519 116 116 116 190 190 190 F F F 26 520-539 116 116 116 190 190 190 F F F 27 540-559 116 116 116 188 188 188 F F F 28 560-579 113 113 113 187 187 187 T T T 29 580-599 113 113 113 187 187 187 F F F Max gen: 116, Max flush: 190 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 187 187 187 F F F 1 020-039 110 110 110 184 184 184 F F F 2 040-059 109 109 109 182 182 182 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 113 113 113 187 187 187 F F F 5 100-119 113 113 113 187 187 187 F F F 6 120-139 112 112 112 186 186 186 F F F 7 140-159 116 116 116 188 188 188 F F F 8 160-179 117 117 117 191 191 191 F F F 9 180-199 117 117 117 191 191 191 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 113 113 113 187 187 187 F F F 12 240-259 113 113 113 187 187 187 F F F 13 260-279 112 112 112 186 186 186 F F F 14 280-299 113 113 113 187 187 187 F F F 15 300-319 113 113 113 187 187 187 F F F 16 320-339 113 113 113 187 187 187 F F F 17 340-359 116 116 116 190 190 190 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 113 113 113 187 187 187 F F F 20 400-419 116 116 116 190 190 190 F F F 21 420-439 112 112 112 186 186 186 F F F 22 440-459 116 116 116 190 190 190 F F F 23 460-479 116 116 116 189 189 189 F F F 24 480-499 108 108 108 180 180 180 F F F 25 500-519 116 116 116 190 190 190 F F F 26 520-539 116 116 116 190 190 190 F F F 27 540-559 116 116 116 188 188 188 F F F 28 560-579 117 117 117 191 191 191 F F F 29 580-599 113 113 113 187 187 187 F F F Max gen: 117, Max flush: 191 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:07.971526671Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:08.014372785Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.014768831Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.046225068Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.046860535Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.046894182Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.046908036Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.046922039Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660} {"msg":"Crucible 12398ff0-6e04-41d3-ba75-58aec323b1e6 has session id: 50d9fcd7-1a47-4b88-b97e-abd7a04b463e","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.047544753Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Upstairs opts: Upstairs UUID: 12398ff0-6e04-41d3-ba75-58aec323b1e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.047585162Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Crucible stats registered with UUID: 12398ff0-6e04-41d3-ba75-58aec323b1e6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.047602505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:08.047615548Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048050682Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048076008Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660} {"msg":"12398ff0-6e04-41d3-ba75-58aec323b1e6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048120627Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048137161Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"0","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048165097Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"0","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.04817927Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"1","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048194863Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"1","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048210747Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"2","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048237432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"2","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048265958Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"io task","client":"2","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048291944Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"io task","client":"0","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048324351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"io task","client":"1","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34068)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048356308Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"io task","client":"2","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38382)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048606062Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"io task","client":"1","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49318)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.048651012Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"io task","client":"0","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.049180366Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"1","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.049362585Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"0","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.049396993Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"2","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.049731865Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"1","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.05007973Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"0","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050250397Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050281864Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"2","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.05063632Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"[0]R flush_numbers[0..12]: [187, 184, 182, 183, 187, 187, 186, 188, 191, 191, 186, 187]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.05077289Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"[0]R generation[0..12]: [113, 110, 109, 110, 113, 113, 112, 116, 117, 117, 112, 113]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050793834Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050809237Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"[1]R flush_numbers[0..12]: [187, 184, 182, 183, 187, 187, 186, 188, 191, 191, 186, 187]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050823701Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"[1]R generation[0..12]: [113, 110, 109, 110, 113, 113, 112, 116, 117, 117, 112, 113]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050837423Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050851056Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"[2]R flush_numbers[0..12]: [187, 184, 182, 183, 187, 187, 186, 188, 191, 191, 186, 187]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.05086894Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"[2]R generation[0..12]: [113, 110, 109, 110, 113, 113, 112, 116, 117, 117, 112, 113]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050883323Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050897196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050909909Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050922722Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050935145Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050947737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"":"downstairs","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.05096042Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.050972122Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"0","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:27:08.050985285Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"1","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.051050429Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"client":"2","":"downstairs","session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.051064673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.051076695Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"12398ff0-6e04-41d3-ba75-58aec323b1e6 is now active with session: 50d9fcd7-1a47-4b88-b97e-abd7a04b463e","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.051089318Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.05110097Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660,"session_id":"50d9fcd7-1a47-4b88-b97e-abd7a04b463e"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.051113013Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660} {"msg":"Activated sub_volume 12398ff0-6e04-41d3-ba75-58aec323b1e6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:08.051126616Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 480 len 16384 data: 8 8 9 11 02/30 Write block 92 len 40960 data: 7 7 5 5 4 7 7 7 5 6 03/30 Read block 409 len 36864 04/30 Write block 254 len 12288 data: 9 8 7 05/30 Read block 116 len 28672 06/30 Read block 175 len 20480 07/30 Read block 151 len 32768 08/30 Write block 34 len 40960 data: 8 10 9 11 12 13 11 10 10 13 09/30 Read block 281 len 24576 10/30 Write block 252 len 40960 data: 9 9 10 9 8 8 9 9 9 8 11/30 Read block 292 len 16384 12/30 Read block 432 len 32768 13/30 Read block 301 len 24576 14/30 Read block 39 len 28672 15/30 Write block 55 len 20480 data: 11 9 9 11 9 16/30 Flush 17/30 Write block 410 len 8192 data: 8 8 18/30 Read block 246 len 8192 19/30 Read block 146 len 32768 20/30 Write block 399 len 4096 data: 11 21/30 Read block 452 len 28672 22/30 Read block 463 len 24576 23/30 Write block 138 len 16384 data: 8 10 9 8 24/30 Flush 25/30 Write block 3 len 36864 data: 8 10 9 11 13 11 9 10 8 26/30 Write block 504 len 28672 data: 9 8 7 8 9 11 12 27/30 Read block 340 len 24576 28/30 Write block 496 len 20480 data: 11 10 10 11 11 29/30 Write block 307 len 40960 data: 10 11 10 6 7 7 8 7 7 9 30/30 Write block 546 len 24576 data: 14 11 10 10 10 10 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:08.072290702Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1660} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 187 187 187 T T T 1 020-039 118 118 118 192 192 192 F F F 2 040-059 118 118 118 192 192 192 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 118 118 118 192 192 192 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 118 118 118 193 193 193 F F F 7 140-159 118 118 118 193 193 193 F F F 8 160-179 117 117 117 191 191 191 F F F 9 180-199 117 117 117 191 191 191 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 113 113 113 187 187 187 F F F 12 240-259 118 118 118 192 192 192 F F F 13 260-279 118 118 118 192 192 192 F F F 14 280-299 113 113 113 187 187 187 F F F 15 300-319 113 113 113 187 187 187 T T T 16 320-339 113 113 113 187 187 187 F F F 17 340-359 116 116 116 190 190 190 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 118 118 118 193 193 193 F F F 21 420-439 112 112 112 186 186 186 F F F 22 440-459 116 116 116 190 190 190 F F F 23 460-479 116 116 116 189 189 189 F F F 24 480-499 118 118 118 192 192 192 T T T 25 500-519 116 116 116 190 190 190 T T T 26 520-539 116 116 116 190 190 190 F F F 27 540-559 116 116 116 188 188 188 T T T 28 560-579 117 117 117 191 191 191 F F F 29 580-599 113 113 113 187 187 187 F F F Max gen: 118, Max flush: 193 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 194 194 194 F F F 1 020-039 118 118 118 192 192 192 F F F 2 040-059 118 118 118 192 192 192 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 118 118 118 192 192 192 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 118 118 118 193 193 193 F F F 7 140-159 118 118 118 193 193 193 F F F 8 160-179 117 117 117 191 191 191 F F F 9 180-199 117 117 117 191 191 191 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 113 113 113 187 187 187 F F F 12 240-259 118 118 118 192 192 192 F F F 13 260-279 118 118 118 192 192 192 F F F 14 280-299 113 113 113 187 187 187 F F F 15 300-319 119 119 119 194 194 194 F F F 16 320-339 113 113 113 187 187 187 F F F 17 340-359 116 116 116 190 190 190 F F F 18 360-379 98 98 98 160 160 160 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 118 118 118 193 193 193 F F F 21 420-439 112 112 112 186 186 186 F F F 22 440-459 116 116 116 190 190 190 F F F 23 460-479 116 116 116 189 189 189 F F F 24 480-499 119 119 119 194 194 194 F F F 25 500-519 119 119 119 194 194 194 F F F 26 520-539 116 116 116 190 190 190 F F F 27 540-559 119 119 119 194 194 194 F F F 28 560-579 117 117 117 191 191 191 F F F 29 580-599 113 113 113 187 187 187 F F F Max gen: 119, Max flush: 194 Begin loop 60 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.342734267Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:12.386394908Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.386756466Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.417370822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.417977092Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.418023733Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.418042877Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.41805749Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668} {"msg":"Crucible ca14fdc2-31a3-450b-8fd1-2250ad49a59a has session id: 9bd41948-5931-473d-8973-692d60ab4e6b","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.418612189Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Upstairs opts: Upstairs UUID: ca14fdc2-31a3-450b-8fd1-2250ad49a59a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.418643946Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Crucible stats registered with UUID: ca14fdc2-31a3-450b-8fd1-2250ad49a59a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.418658579Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:12.418670922Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.4191713Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419193415Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668} {"msg":"ca14fdc2-31a3-450b-8fd1-2250ad49a59a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419259999Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419276693Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"0","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419305339Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"0","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419319252Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"1","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419333115Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"1","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419345828Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"2","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.41940077Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"2","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419436207Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"io task","client":"2","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419463533Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"io task","client":"0","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419491029Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"io task","client":"1","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48613)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419518915Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"io task","client":"0","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50969)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419708226Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"io task","client":"1","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48715)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.419744024Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"io task","client":"2","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.420207414Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"1","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.42023705Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"2","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.420457198Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"0","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.420952124Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"2","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.42098066Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"0","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421011337Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"1","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421570548Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421690834Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"[0]R flush_numbers[0..12]: [194, 192, 192, 183, 192, 192, 193, 193, 191, 191, 186, 187]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421760059Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"[0]R generation[0..12]: [119, 118, 118, 110, 118, 118, 118, 118, 117, 117, 112, 113]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421780633Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421795176Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"[1]R flush_numbers[0..12]: [194, 192, 192, 183, 192, 192, 193, 193, 191, 191, 186, 187]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421808949Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"[1]R generation[0..12]: [119, 118, 118, 110, 118, 118, 118, 118, 117, 117, 112, 113]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421822782Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421836465Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"[2]R flush_numbers[0..12]: [194, 192, 192, 183, 192, 192, 193, 193, 191, 191, 186, 187]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421849848Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"[2]R generation[0..12]: [119, 118, 118, 110, 118, 118, 118, 118, 117, 117, 112, 113]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421863121Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421876604Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421889287Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421902199Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421914852Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421927565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"":"downstairs","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:27:12.421948099Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421968204Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"0","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421981227Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"1","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.421994149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"client":"2","":"downstairs","session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.422007282Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.422019215Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"ca14fdc2-31a3-450b-8fd1-2250ad49a59a is now active with session: 9bd41948-5931-473d-8973-692d60ab4e6b","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.422031687Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.42204327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668,"session_id":"9bd41948-5931-473d-8973-692d60ab4e6b"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.422057313Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668} {"msg":"Activated sub_volume ca14fdc2-31a3-450b-8fd1-2250ad49a59a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:12.422068615Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 489 len 12288 data: 14 14 15 02/30 Read block 385 len 12288 03/30 Flush 04/30 Write block 577 len 12288 data: 11 11 11 05/30 Flush 06/30 Write block 233 len 40960 data: 11 10 12 9 8 9 8 8 7 7 07/30 Read block 481 len 8192 08/30 Write block 564 len 16384 data: 12 11 10 12 09/30 Read block 275 len 20480 10/30 Write block 132 len 28672 data: 6 6 6 7 7 6 9 11/30 Read block 28 len 12288 12/30 Write block 531 len 8192 data: 11 10 13/30 Read block 543 len 20480 14/30 Read block 368 len 12288 15/30 Read block 390 len 24576 16/30 Write block 240 len 8192 data: 9 8 17/30 Read block 217 len 16384 18/30 Write block 409 len 20480 data: 9 9 9 7 7 19/30 Write block 427 len 40960 data: 10 12 11 12 11 11 11 10 8 7 20/30 Read block 232 len 24576 21/30 Read block 366 len 4096 22/30 Write block 356 len 20480 data: 7 8 10 10 11 23/30 Read block 457 len 16384 24/30 Read block 288 len 4096 25/30 Read block 381 len 12288 26/30 Write block 262 len 24576 data: 8 11 12 14 12 11 27/30 Write block 193 len 24576 data: 6 10 9 8 10 9 28/30 Write block 410 len 20480 data: 10 10 8 8 7 29/30 Read block 176 len 12288 30/30 Read block 198 len 16384 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 10 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 25 24 24 73 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:12.441860886Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1668} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 194 194 194 F F F 1 020-039 118 118 118 192 192 192 F F F 2 040-059 118 118 118 192 192 192 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 118 118 118 192 192 192 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 118 118 118 193 193 193 T T T 7 140-159 118 118 118 193 193 193 F F F 8 160-179 117 117 117 191 191 191 F F F 9 180-199 117 117 117 191 191 191 T T T 10 200-219 112 112 112 186 186 186 F F F 11 220-239 113 113 113 187 187 187 T T T 12 240-259 118 118 118 192 192 192 T T T 13 260-279 118 118 118 192 192 192 T T T 14 280-299 113 113 113 187 187 187 F F F 15 300-319 119 119 119 194 194 194 F F F 16 320-339 113 113 113 187 187 187 F F F 17 340-359 116 116 116 190 190 190 T T T 18 360-379 98 98 98 160 160 160 T T T 19 380-399 118 118 118 193 193 193 F F F 20 400-419 118 118 118 193 193 193 T T T 21 420-439 112 112 112 186 186 186 T T T 22 440-459 116 116 116 190 190 190 F F F 23 460-479 116 116 116 189 189 189 F F F 24 480-499 120 120 120 195 195 195 F F F 25 500-519 119 119 119 194 194 194 F F F 26 520-539 116 116 116 190 190 190 T T T 27 540-559 119 119 119 194 194 194 F F F 28 560-579 120 120 120 196 196 196 T T T 29 580-599 113 113 113 187 187 187 F F F Max gen: 120, Max flush: 196 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 194 194 194 F F F 1 020-039 118 118 118 192 192 192 F F F 2 040-059 118 118 118 192 192 192 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 118 118 118 192 192 192 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 121 121 121 197 197 197 F F F 7 140-159 118 118 118 193 193 193 F F F 8 160-179 117 117 117 191 191 191 F F F 9 180-199 121 121 121 197 197 197 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 121 121 121 197 197 197 F F F 12 240-259 121 121 121 197 197 197 F F F 13 260-279 121 121 121 197 197 197 F F F 14 280-299 113 113 113 187 187 187 F F F 15 300-319 119 119 119 194 194 194 F F F 16 320-339 113 113 113 187 187 187 F F F 17 340-359 121 121 121 197 197 197 F F F 18 360-379 121 121 121 197 197 197 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 121 121 121 197 197 197 F F F 21 420-439 121 121 121 197 197 197 F F F 22 440-459 116 116 116 190 190 190 F F F 23 460-479 116 116 116 189 189 189 F F F 24 480-499 120 120 120 195 195 195 F F F 25 500-519 119 119 119 194 194 194 F F F 26 520-539 121 121 121 197 197 197 F F F 27 540-559 119 119 119 194 194 194 F F F 28 560-579 121 121 121 197 197 197 F F F 29 580-599 113 113 113 187 187 187 F F F Max gen: 121, Max flush: 197 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.741397758Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.785289618Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.786220739Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.786261157Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.78627547Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.786289483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676} {"msg":"Crucible eb9d751c-0d7a-4aef-bbd7-b7e5d06677ad has session id: 53c1235e-e5a5-431e-9733-bf6463bfc615","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.786920479Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Upstairs opts: Upstairs UUID: eb9d751c-0d7a-4aef-bbd7-b7e5d06677ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.786973831Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Crucible stats registered with UUID: eb9d751c-0d7a-4aef-bbd7-b7e5d06677ad","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.78701351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:16.787031543Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787391611Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787420377Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676} {"msg":"eb9d751c-0d7a-4aef-bbd7-b7e5d06677ad active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787484721Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787505226Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"0","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787537923Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"0","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787555516Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"1","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.78757295Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"1","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787600156Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"2","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787628832Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"2","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787654888Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"io task","client":"2","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787680643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"io task","client":"0","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787706899Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"ds_connection connected from Ok(127.0.0.1:57996)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787730944Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"io task","client":"2","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:16.787764922Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:16.787795608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:16.787823914Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"ds_connection connected from Ok(127.0.0.1:52497)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.787916854Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"io task","client":"0","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.78799024Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.788483017Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"2","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.788513313Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"0","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.789179447Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"0","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.789210564Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"2","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.789808953Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:16.789836559Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.788794724Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"ds_connection connected from Ok(127.0.0.1:45413)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.788861569Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"io task","client":"1","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.790109858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"1","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.790640023Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"1","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"[0]R flush_numbers[0..12]: [194, 192, 192, 183, 192, 192, 197, 193, 191, 197, 186, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791223829Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"[0]R generation[0..12]: [119, 118, 118, 110, 118, 118, 121, 118, 117, 121, 112, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791245173Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791260327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"[1]R flush_numbers[0..12]: [194, 192, 192, 183, 192, 192, 197, 193, 191, 197, 186, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.79127436Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"[1]R generation[0..12]: [119, 118, 118, 110, 118, 118, 121, 118, 117, 121, 112, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791287922Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791301636Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"[2]R flush_numbers[0..12]: [194, 192, 192, 183, 192, 192, 197, 193, 191, 197, 186, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791315199Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"[2]R generation[0..12]: [119, 118, 118, 110, 118, 118, 121, 118, 117, 121, 112, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791328831Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791342574Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791355057Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.79136782Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791380242Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791393015Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"":"downstairs","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791409449Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791421581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","Wait for a query_work_queue command to finish before sending IO pid":1676,"client":"0","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791442966Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"1","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791455759Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"client":"2","":"downstairs","session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791468732Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791480834Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"eb9d751c-0d7a-4aef-bbd7-b7e5d06677ad is now active with session: 53c1235e-e5a5-431e-9733-bf6463bfc615","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791493437Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791505029Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676,"session_id":"53c1235e-e5a5-431e-9733-bf6463bfc615"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791516932Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676} {"msg":"Activated sub_volume eb9d751c-0d7a-4aef-bbd7-b7e5d06677ad","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:26.791527864Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 502 len 32768 data: 12 12 10 9 8 9 10 12 02/30 Read block 273 len 28672 03/30 Write block 294 len 4096 data: 8 04/30 Write block 85 len 32768 data: 7 8 7 7 6 8 8 8 05/30 Write block 321 len 36864 data: 8 8 9 9 10 9 10 7 8 06/30 Read block 566 len 40960 07/30 Read block 30 len 40960 08/30 Write block 128 len 20480 data: 4 4 4 5 7 09/30 Read block 25 len 20480 10/30 Read block 317 len 4096 11/30 Read block 379 len 4096 12/30 Read block 269 len 4096 13/30 Read block 135 len 12288 14/30 Write block 546 len 16384 data: 15 12 11 11 15/30 Write block 5 len 8192 data: 10 12 16/30 Read block 577 len 20480 17/30 Read block 46 len 32768 18/30 Read block 281 len 16384 19/30 Read block 44 len 20480 20/30 Read block 347 len 24576 21/30 Write block 435 len 24576 data: 9 8 6 5 7 10 22/30 Flush 23/30 Flush 24/30 Write block 543 len 4096 data: 13 25/30 Flush 26/30 Write block 335 len 40960 data: 17 14 10 10 11 9 7 9 7 8 27/30 Read block 436 len 4096 28/30 Read block 137 len 12288 29/30 Read block 146 len 8192 30/30 Write block 456 len 24576 data: 10 11 10 9 9 9 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:26.812467812Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1676} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 118 118 118 192 192 192 F F F 2 040-059 118 118 118 192 192 192 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 122 122 122 198 198 198 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 122 122 122 198 198 198 F F F 7 140-159 118 118 118 193 193 193 F F F 8 160-179 117 117 117 191 191 191 F F F 9 180-199 121 121 121 197 197 197 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 121 121 121 197 197 197 F F F 12 240-259 121 121 121 197 197 197 F F F 13 260-279 121 121 121 197 197 197 F F F 14 280-299 122 122 122 198 198 198 F F F 15 300-319 119 119 119 194 194 194 F F F 16 320-339 122 122 122 198 198 198 T T T 17 340-359 121 121 121 197 197 197 T T T 18 360-379 121 121 121 197 197 197 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 121 121 121 197 197 197 F F F 21 420-439 122 122 122 198 198 198 F F F 22 440-459 122 122 122 198 198 198 T T T 23 460-479 116 116 116 189 189 189 T T T 24 480-499 120 120 120 195 195 195 F F F 25 500-519 122 122 122 198 198 198 F F F 26 520-539 121 121 121 197 197 197 F F F 27 540-559 122 122 122 200 200 200 F F F 28 560-579 121 121 121 197 197 197 F F F 29 580-599 113 113 113 187 187 187 F F F Max gen: 122, Max flush: 200 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 118 118 118 192 192 192 F F F 2 040-059 118 118 118 192 192 192 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 122 122 122 198 198 198 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 122 122 122 198 198 198 F F F 7 140-159 118 118 118 193 193 193 F F F 8 160-179 117 117 117 191 191 191 F F F 9 180-199 121 121 121 197 197 197 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 121 121 121 197 197 197 F F F 12 240-259 121 121 121 197 197 197 F F F 13 260-279 121 121 121 197 197 197 F F F 14 280-299 122 122 122 198 198 198 F F F 15 300-319 119 119 119 194 194 194 F F F 16 320-339 123 123 123 201 201 201 F F F 17 340-359 123 123 123 201 201 201 F F F 18 360-379 121 121 121 197 197 197 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 121 121 121 197 197 197 F F F 21 420-439 122 122 122 198 198 198 F F F 22 440-459 123 123 123 201 201 201 F F F 23 460-479 123 123 123 201 201 201 F F F 24 480-499 120 120 120 195 195 195 F F F 25 500-519 122 122 122 198 198 198 F F F 26 520-539 121 121 121 197 197 197 F F F 27 540-559 122 122 122 200 200 200 F F F 28 560-579 121 121 121 197 197 197 F F F 29 580-599 113 113 113 187 187 187 F F F Max gen: 123, Max flush: 201 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.053088601Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:31.095960432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.096400476Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.127705461Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.128422016Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.128457713Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.128471996Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.12848624Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684} {"msg":"Crucible 5887100e-4358-4b6f-a57d-e623412997de has session id: 6b22f73b-f76a-45d5-93b8-765940d08e55","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129192672Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Upstairs opts: Upstairs UUID: 5887100e-4358-4b6f-a57d-e623412997de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129234441Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Crucible stats registered with UUID: 5887100e-4358-4b6f-a57d-e623412997de","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129258476Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:31.129278101Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129698321Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129724927Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684} {"msg":"5887100e-4358-4b6f-a57d-e623412997de active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129766726Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129783579Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"0","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129812676Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"0","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.1298322Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"1","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129857986Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"1","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129909857Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"2","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129933862Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"2","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129956907Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"io task","client":"2","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.129982132Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"io task","client":"0","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.130006958Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"io task","client":"1","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"ds_connection connected from Ok(127.0.0.1:48914)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.130032163Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"io task","client":"2","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"ds_connection connected from Ok(127.0.0.1:53553)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.130236227Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"io task","client":"0","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"ds_connection connected from Ok(127.0.0.1:49235)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.130276256Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"io task","client":"1","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.130628682Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"2","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.130781045Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"1","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.130900791Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"0","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.131319151Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"2","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.131348127Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"1","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.131475505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"0","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.131976503Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132013041Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"[0]R flush_numbers[0..12]: [198, 192, 192, 183, 198, 192, 198, 193, 191, 197, 186, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132121494Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"[0]R generation[0..12]: [122, 118, 118, 110, 122, 118, 122, 118, 117, 121, 112, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132143199Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132157902Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"[1]R flush_numbers[0..12]: [198, 192, 192, 183, 198, 192, 198, 193, 191, 197, 186, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132171875Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"[1]R generation[0..12]: [122, 118, 118, 110, 122, 118, 122, 118, 117, 121, 112, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132185488Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132199831Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"[2]R flush_numbers[0..12]: [198, 192, 192, 183, 198, 192, 198, 193, 191, 197, 186, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132216884Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"[2]R generation[0..12]: [122, 118, 118, 110, 122, 118, 122, 118, 117, 121, 112, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132230787Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132244781Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132257953Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132270936Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132283589Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:27:31.132296102Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"":"downstairs","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132313816Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132325338Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"0","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132338331Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"1","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132350913Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"client":"2","":"downstairs","session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132363616Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132375489Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"5887100e-4358-4b6f-a57d-e623412997de is now active with session: 6b22f73b-f76a-45d5-93b8-765940d08e55","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132387992Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132399554Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684,"session_id":"6b22f73b-f76a-45d5-93b8-765940d08e55"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132411397Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684} {"msg":"Activated sub_volume 5887100e-4358-4b6f-a57d-e623412997de","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:31.132424169Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 528 len 20480 data: 11 11 9 12 11 02/30 Read block 335 len 4096 03/30 Read block 115 len 40960 04/30 Read block 159 len 28672 05/30 Write block 35 len 20480 data: 11 10 12 13 14 06/30 Flush 07/30 Write block 431 len 40960 data: 12 12 12 11 10 9 7 6 8 11 08/30 Read block 42 len 16384 09/30 Read block 357 len 24576 10/30 Flush 11/30 Flush 12/30 Write block 296 len 24576 data: 11 12 12 9 11 11 13/30 Write block 325 len 8192 data: 11 10 14/30 Read block 481 len 4096 15/30 Read block 53 len 4096 16/30 Write block 154 len 40960 data: 15 13 13 13 12 13 12 11 8 12 17/30 Write block 276 len 32768 data: 10 7 8 7 10 11 11 11 18/30 Flush 19/30 Flush 20/30 Write block 517 len 24576 data: 14 14 12 9 10 11 21/30 Read block 316 len 36864 22/30 Read block 235 len 20480 23/30 Write block 559 len 36864 data: 11 14 13 15 14 13 12 11 13 24/30 Read block 76 len 16384 25/30 Write block 402 len 40960 data: 9 10 10 11 12 14 12 10 11 11 26/30 Write block 342 len 12288 data: 10 8 9 27/30 Read block 267 len 36864 28/30 Write block 310 len 32768 data: 7 8 8 9 8 8 10 9 29/30 Read block 288 len 24576 30/30 Read block 233 len 36864 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 6 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 11 11 10 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:31.154193967Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1684} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 118 118 118 192 192 192 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 122 122 122 198 198 198 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 122 122 122 198 198 198 F F F 7 140-159 124 124 124 205 205 205 F F F 8 160-179 124 124 124 205 205 205 F F F 9 180-199 121 121 121 197 197 197 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 121 121 121 197 197 197 F F F 12 240-259 121 121 121 197 197 197 F F F 13 260-279 124 124 124 205 205 205 F F F 14 280-299 124 124 124 205 205 205 F F F 15 300-319 124 124 124 205 205 205 T T T 16 320-339 124 124 124 205 205 205 F F F 17 340-359 123 123 123 201 201 201 T T T 18 360-379 121 121 121 197 197 197 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 121 121 121 197 197 197 T T T 21 420-439 124 124 124 203 203 203 F F F 22 440-459 124 124 124 203 203 203 F F F 23 460-479 123 123 123 201 201 201 F F F 24 480-499 120 120 120 195 195 195 F F F 25 500-519 122 122 122 198 198 198 T T T 26 520-539 124 124 124 202 202 202 T T T 27 540-559 122 122 122 200 200 200 T T T 28 560-579 121 121 121 197 197 197 T T T 29 580-599 113 113 113 187 187 187 F F F Max gen: 124, Max flush: 205 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 118 118 118 192 192 192 F F F 3 060-079 110 110 110 183 183 183 F F F 4 080-099 122 122 122 198 198 198 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 122 122 122 198 198 198 F F F 7 140-159 124 124 124 205 205 205 F F F 8 160-179 124 124 124 205 205 205 F F F 9 180-199 121 121 121 197 197 197 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 121 121 121 197 197 197 F F F 12 240-259 121 121 121 197 197 197 F F F 13 260-279 124 124 124 205 205 205 F F F 14 280-299 124 124 124 205 205 205 F F F 15 300-319 125 125 125 206 206 206 F F F 16 320-339 124 124 124 205 205 205 F F F 17 340-359 125 125 125 206 206 206 F F F 18 360-379 121 121 121 197 197 197 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 125 125 125 206 206 206 F F F 21 420-439 124 124 124 203 203 203 F F F 22 440-459 124 124 124 203 203 203 F F F 23 460-479 123 123 123 201 201 201 F F F 24 480-499 120 120 120 195 195 195 F F F 25 500-519 125 125 125 206 206 206 F F F 26 520-539 125 125 125 206 206 206 F F F 27 540-559 125 125 125 206 206 206 F F F 28 560-579 125 125 125 206 206 206 F F F 29 580-599 113 113 113 187 187 187 F F F Max gen: 125, Max flush: 206 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.437668473Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:35.482618812Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.483019098Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.514423784Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.515164774Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.515207013Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.515229168Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.515248972Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692} {"msg":"Crucible 557a761c-2531-481b-841d-c74fd076b4dd has session id: 78a305a8-4991-4e3b-87c1-9f9299a8fa27","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.515779287Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Upstairs opts: Upstairs UUID: 557a761c-2531-481b-841d-c74fd076b4dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.515822236Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Crucible stats registered with UUID: 557a761c-2531-481b-841d-c74fd076b4dd","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.51583855Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:35.515852333Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516276544Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516309621Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692} {"msg":"557a761c-2531-481b-841d-c74fd076b4dd active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516362853Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516389398Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"0","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516422385Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"0","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516437829Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"1","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516452422Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"1","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516470066Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"2","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516484879Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"2","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516497682Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"io task","client":"2","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516511235Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"io task","client":"0","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516524558Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"io task","client":"1","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"ds_connection connected from Ok(127.0.0.1:56048)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516660047Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"io task","client":"1","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"ds_connection connected from Ok(127.0.0.1:54326)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.516871782Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"io task","client":"2","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"ds_connection connected from Ok(127.0.0.1:38558)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.51690579Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"io task","client":"0","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.517574904Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"0","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.517614233Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"2","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.517655822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"1","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.518224974Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"1","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.518382819Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"0","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.518523149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"2","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.518823874Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519004823Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"[0]R flush_numbers[0..12]: [198, 202, 192, 183, 198, 192, 198, 205, 205, 197, 186, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519145043Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"[0]R generation[0..12]: [122, 124, 118, 110, 122, 118, 122, 124, 124, 121, 112, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519172789Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519189082Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"[1]R flush_numbers[0..12]: [198, 202, 192, 183, 198, 192, 198, 205, 205, 197, 186, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519203626Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"[1]R generation[0..12]: [122, 124, 118, 110, 122, 118, 122, 124, 124, 121, 112, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519217369Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519231202Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"[2]R flush_numbers[0..12]: [198, 202, 192, 183, 198, 192, 198, 205, 205, 197, 186, 197]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519244554Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"[2]R generation[0..12]: [122, 124, 118, 110, 122, 118, 122, 124, 124, 121, 112, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519257887Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.51927163Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519284543Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519297686Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519310239Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"All extents match","v":0,"nameWait for a query_work_queue command to finish before sending IO ":"crucible","level":30,"time":"2025-03-17T22:27:35.519331523Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"":"downstairs","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519344626Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519356409Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"0","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519369101Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"1","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519381674Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"client":"2","":"downstairs","session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519394317Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519406209Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"557a761c-2531-481b-841d-c74fd076b4dd is now active with session: 78a305a8-4991-4e3b-87c1-9f9299a8fa27","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519418482Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519430174Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692,"session_id":"78a305a8-4991-4e3b-87c1-9f9299a8fa27"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.519441857Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692} {"msg":"Activated sub_volume 557a761c-2531-481b-841d-c74fd076b4dd","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:35.51945259Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 319 len 40960 data: 7 9 9 9 10 10 12 11 11 8 02/30 Read block 222 len 40960 03/30 Write block 43 len 40960 data: 14 16 12 13 13 16 16 14 11 12 04/30 Write block 375 len 8192 data: 9 6 05/30 Read block 492 len 24576 06/30 Read block 559 len 24576 07/30 Flush 08/30 Write block 88 len 16384 data: 8 7 9 9 09/30 Read block 336 len 36864 10/30 Write block 126 len 24576 data: 5 5 5 5 5 6 11/30 Read block 331 len 16384 12/30 Read block 198 len 4096 13/30 Write block 339 len 40960 data: 12 10 8 11 9 10 8 6 6 7 14/30 Read block 36 len 16384 15/30 Read block 292 len 12288 16/30 Write block 128 len 28672 data: 6 6 6 7 8 7 7 17/30 Write block 576 len 28672 data: 9 12 12 12 7 8 8 18/30 Write block 237 len 4096 data: 9 19/30 Read block 218 len 36864 20/30 Write block 563 len 28672 data: 15 14 13 12 14 13 13 21/30 Write block 74 len 4096 data: 15 22/30 Read block 292 len 32768 23/30 Read block 189 len 8192 24/30 Read block 81 len 4096 25/30 Read block 467 len 8192 26/30 Read block 550 len 24576 27/30 Read block 352 len 40960 28/30 Write block 521 len 36864 data: 11 12 12 14 13 14 14 12 12 29/30 Read block 397 len 28672 30/30 Read block 247 len 8192 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 4 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:35.541488864Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1692} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 126 126 126 207 207 207 F F F 3 060-079 110 110 110 183 183 183 T T T 4 080-099 122 122 122 198 198 198 T T T 5 100-119 118 118 118 192 192 192 F F F 6 120-139 122 122 122 198 198 198 T T T 7 140-159 124 124 124 205 205 205 F F F 8 160-179 124 124 124 205 205 205 F F F 9 180-199 121 121 121 197 197 197 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 121 121 121 197 197 197 T T T 12 240-259 121 121 121 197 197 197 F F F 13 260-279 124 124 124 205 205 205 F F F 14 280-299 124 124 124 205 205 205 F F F 15 300-319 126 126 126 207 207 207 F F F 16 320-339 126 126 126 207 207 207 T T T 17 340-359 125 125 125 206 206 206 T T T 18 360-379 126 126 126 207 207 207 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 125 125 125 206 206 206 F F F 21 420-439 124 124 124 203 203 203 F F F 22 440-459 124 124 124 203 203 203 F F F 23 460-479 123 123 123 201 201 201 F F F 24 480-499 120 120 120 195 195 195 F F F 25 500-519 125 125 125 206 206 206 F F F 26 520-539 125 125 125 206 206 206 T T T 27 540-559 125 125 125 206 206 206 F F F 28 560-579 125 125 125 206 206 206 T T T 29 580-599 113 113 113 187 187 187 T T T Max gen: 126, Max flush: 207 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 126 126 126 207 207 207 F F F 3 060-079 127 127 127 208 208 208 F F F 4 080-099 127 127 127 208 208 208 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 127 127 127 208 208 208 F F F 7 140-159 124 124 124 205 205 205 F F F 8 160-179 124 124 124 205 205 205 F F F 9 180-199 121 121 121 197 197 197 F F F 10 200-219 112 112 112 186 186 186 F F F 11 220-239 127 127 127 208 208 208 F F F 12 240-259 121 121 121 197 197 197 F F F 13 260-279 124 124 124 205 205 205 F F F 14 280-299 124 124 124 205 205 205 F F F 15 300-319 126 126 126 207 207 207 F F F 16 320-339 127 127 127 208 208 208 F F F 17 340-359 127 127 127 208 208 208 F F F 18 360-379 126 126 126 207 207 207 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 125 125 125 206 206 206 F F F 21 420-439 124 124 124 203 203 203 F F F 22 440-459 124 124 124 203 203 203 F F F 23 460-479 123 123 123 201 201 201 F F F 24 480-499 120 120 120 195 195 195 F F F 25 500-519 125 125 125 206 206 206 F F F 26 520-539 127 127 127 208 208 208 F F F 27 540-559 125 125 125 206 206 206 F F F 28 560-579 127 127 127 208 208 208 F F F 29 580-599 127 127 127 208 208 208 F F F Max gen: 127, Max flush: 208 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.83265796Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.87714935Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.877772464Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.877819174Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.877834507Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.877849271Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700} {"msg":"Crucible 91c396c1-2bb0-47df-b3c1-259d0a4a9f19 has session id: deeb132b-c504-4766-ba28-26b717cb64e5","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.878416023Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Upstairs opts: Upstairs UUID: 91c396c1-2bb0-47df-b3c1-259d0a4a9f19, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.878460863Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Crucible stats registered with UUID: 91c396c1-2bb0-47df-b3c1-259d0a4a9f19","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.878489209Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:39.878510953Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.879021324Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.879053461Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700} {"msg":"91c396c1-2bb0-47df-b3c1-259d0a4a9f19 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.879108953Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.879134508Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"0","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.879166065Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"0","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.87918999Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"1","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.879216696Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"1","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.879239551Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"2","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.879262306Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"2","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.87928369Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"io task","client":"2","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.879306645Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"io task","client":"0","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.87932971Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"io task","client":"1","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:40669)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.879354206Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"io task","client":"2","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:39800)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.879383072Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"io task","client":"0","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:47523)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.87965299Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"io task","client":"1","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.8802117Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"0","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.88025366Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"2","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.880454483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"1","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881015754Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"0","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881053672Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"2","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881081968Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"1","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881568743Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881685098Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"[0]R flush_numbers[0..12]: [198, 202, 207, 208, 208, 192, 208, 205, 205, 197, 186, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881720346Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"[0]R generation[0..12]: [122, 124, 126, 127, 127, 118, 127, 124, 124, 121, 112, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.88173694Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881751543Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"[1]R flush_numbers[0..12]: [198, 202, 207, 208, 208, 192, 208, 205, 205, 197, 186, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881765586Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"[1]R generation[0..12]: [122, 124, 126, 127, 127, 118, 127, 124, 124, 121, 112, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881782099Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881796313Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"[2]R flush_numbers[0..12]: [198, 202, 207, 208, 208, 192, 208, 205, 205, 197, 186, 208]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881809825Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"[2]R generation[0..12]: [122, 124, 126, 127, 127, 118, 127, 124, 124, 121, 112, 127]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881823218Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881836591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881849904Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881863357Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881885452Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881906656Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"":"downstairs","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881929051Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881945194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"0","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.88196896Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"1","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.881995005Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"client":"2","":"downstairs","session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.882020791Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v"Wait for a query_work_queue command to finish before sending IO :0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.882052428Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"91c396c1-2bb0-47df-b3c1-259d0a4a9f19 is now active with session: deeb132b-c504-4766-ba28-26b717cb64e5","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.882081844Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.882105989Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700,"session_id":"deeb132b-c504-4766-ba28-26b717cb64e5"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.882129524Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700} {"msg":"Activated sub_volume 91c396c1-2bb0-47df-b3c1-259d0a4a9f19","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:39.882152009Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 595 len 16384 data: 4 5 5 5 02/30 Read block 65 len 32768 03/30 Read block 243 len 36864 04/30 Read block 52 len 28672 05/30 Read block 540 len 24576 06/30 Read block 553 len 36864 07/30 Read block 450 len 20480 08/30 Read block 377 len 36864 09/30 Read block 584 len 8192 10/30 Write block 439 len 24576 data: 9 12 9 11 11 10 11/30 Write block 590 len 28672 data: 10 11 9 9 7 5 6 12/30 Write block 285 len 32768 data: 13 12 9 9 7 10 8 8 13/30 Read block 41 len 24576 14/30 Read block 410 len 36864 15/30 Read block 263 len 4096 16/30 Read block 325 len 32768 17/30 Read block 231 len 32768 18/30 Flush 19/30 Write block 224 len 8192 data: 10 7 20/30 Write block 204 len 16384 data: 9 12 11 9 21/30 Flush 22/30 Write block 149 len 16384 data: 12 10 14 12 23/30 Read block 378 len 12288 24/30 Write block 152 len 4096 data: 13 25/30 Write block 193 len 40960 data: 7 11 10 9 11 10 9 13 15 12 26/30 Read block 369 len 32768 27/30 Write block 442 len 36864 data: 12 12 11 11 13 11 9 9 7 28/30 Read block 506 len 36864 29/30 Read block 469 len 8192 30/30 Read block 129 len 16384 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:39.903026133Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1700} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 126 126 126 207 207 207 F F F 3 060-079 127 127 127 208 208 208 F F F 4 080-099 127 127 127 208 208 208 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 127 127 127 208 208 208 F F F 7 140-159 124 124 124 205 205 205 T T T 8 160-179 124 124 124 205 205 205 F F F 9 180-199 121 121 121 197 197 197 T T T 10 200-219 128 128 128 210 210 210 T T T 11 220-239 128 128 128 210 210 210 F F F 12 240-259 121 121 121 197 197 197 F F F 13 260-279 124 124 124 205 205 205 F F F 14 280-299 128 128 128 209 209 209 F F F 15 300-319 126 126 126 207 207 207 F F F 16 320-339 127 127 127 208 208 208 F F F 17 340-359 127 127 127 208 208 208 F F F 18 360-379 126 126 126 207 207 207 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 125 125 125 206 206 206 F F F 21 420-439 128 128 128 209 209 209 F F F 22 440-459 128 128 128 209 209 209 T T T 23 460-479 123 123 123 201 201 201 F F F 24 480-499 120 120 120 195 195 195 F F F 25 500-519 125 125 125 206 206 206 F F F 26 520-539 127 127 127 208 208 208 F F F 27 540-559 125 125 125 206 206 206 F F F 28 560-579 127 127 127 208 208 208 F F F 29 580-599 128 128 128 209 209 209 F F F Max gen: 128, Max flush: 210 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 126 126 126 207 207 207 F F F 3 060-079 127 127 127 208 208 208 F F F 4 080-099 127 127 127 208 208 208 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 127 127 127 208 208 208 F F F 7 140-159 129 129 129 211 211 211 F F F 8 160-179 124 124 124 205 205 205 F F F 9 180-199 129 129 129 211 211 211 F F F 10 200-219 129 129 129 211 211 211 F F F 11 220-239 128 128 128 210 210 210 F F F 12 240-259 121 121 121 197 197 197 F F F 13 260-279 124 124 124 205 205 205 F F F 14 280-299 128 128 128 209 209 209 F F F 15 300-319 126 126 126 207 207 207 F F F 16 320-339 127 127 127 208 208 208 F F F 17 340-359 127 127 127 208 208 208 F F F 18 360-379 126 126 126 207 207 207 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 125 125 125 206 206 206 F F F 21 420-439 128 128 128 209 209 209 F F F 22 440-459 129 129 129 211 211 211 F F F 23 460-479 123 123 123 201 201 201 F F F 24 480-499 120 120 120 195 195 195 F F F 25 500-519 125 125 125 206 206 206 F F F 26 520-539 127 127 127 208 208 208 F F F 27 540-559 125 125 125 206 206 206 F F F 28 560-579 127 127 127 208 208 208 F F F 29 580-599 128 128 128 209 209 209 F F F Max gen: 129, Max flush: 211 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.137459828Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.18242306Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.183139294Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.183184094Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.183202938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.183217741Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708} {"msg":"Crucible d06e13ef-729f-4c76-8e87-070f605ffb57 has session id: bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.183673239Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Upstairs opts: Upstairs UUID: d06e13ef-729f-4c76-8e87-070f605ffb57, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.183709577Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Crucible stats registered with UUID: d06e13ef-729f-4c76-8e87-070f605ffb57","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.183734433Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:44.183758418Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184260186Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184284431Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708} {"msg":"d06e13ef-729f-4c76-8e87-070f605ffb57 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184360938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184378772Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"0","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184405348Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"0","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184425202Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"1","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184451658Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"1","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184482184Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"2","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184503529Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"2","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184517052Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"io task","client":"2","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184530855Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"io task","client":"0","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184545968Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"io task","client":"1","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:57962)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184745831Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"io task","client":"0","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:43217)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184784619Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"io task","client":"2","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:43966)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.184811505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"io task","client":"1","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.185546214Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"2","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.185582342Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"0","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.186250946Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"0","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.186291584Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"2","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.186621786Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"1","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.186909978Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.186936304Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187238369Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"1","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"[0]R flush_numbers[0..12]: [198, 202, 207, 208, 208, 192, 208, 211, 205, 211, 211, 210]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187767553Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"[0]R generation[0..12]: [122, 124, 126, 127, 127, 118, 127, 129, 124, 129, 129, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187787117Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.18780143Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"[1]R flush_numbers[0..12]: [198, 202, 207, 208, 208, 192, 208, 211, 205, 211, 211, 210]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187815113Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"[1]R generation[0..12]: [122, 124, 126, 127, 127, 118, 127, 129, 124, 129, 129, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187828316Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187841559Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"[2]R flush_numbers[0..12]: [198, 202, 207, 208, 208, 192, 208, 211, 205, 211, 211, 210]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187854752Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"[2]R generation[0..12]: [122, 124, 126, 127, 127, 118, 127, 129, 124, 129, 129, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187867985Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187881618Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.18789412Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187906743Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187919086Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:27:44.187931528Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"":"downstairs","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187952203Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187963835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"0","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187976708Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"1","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.187989531Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"client":"2","":"downstairs","session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.188018747Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.18803182Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"d06e13ef-729f-4c76-8e87-070f605ffb57 is now active with session: bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.188044193Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.188055815Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708,"session_id":"bdcd7f37-1ccd-4fda-ad9a-2f9a3cb029ff"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.188067578Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708} {"msg":"Activated sub_volume d06e13ef-729f-4c76-8e87-070f605ffb57","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:44.18807844Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 145 len 8192 data: 10 10 02/30 Flush 03/30 Write block 461 len 4096 data: 10 04/30 Write block 409 len 28672 data: 11 12 12 9 9 8 5 05/30 Write block 233 len 16384 data: 12 11 13 10 06/30 Write block 482 len 8192 data: 10 12 07/30 Read block 43 len 40960 08/30 Flush 09/30 Write block 418 len 8192 data: 6 6 10/30 Read block 460 len 20480 11/30 Read block 34 len 12288 12/30 Write block 257 len 20480 data: 9 10 10 10 9 13/30 Read block 488 len 32768 14/30 Write block 73 len 32768 data: 12 16 14 12 10 9 5 6 15/30 Read block 367 len 8192 16/30 Read block 161 len 36864 17/30 Write block 175 len 8192 data: 6 9 18/30 Read block 327 len 12288 19/30 Write block 96 len 16384 data: 5 8 8 8 20/30 Write block 327 len 8192 data: 12 9 21/30 Write block 78 len 4096 data: 10 22/30 Read block 70 len 40960 23/30 Read block 557 len 4096 24/30 Flush 25/30 Write block 139 len 28672 data: 11 10 9 8 10 10 11 26/30 Write block 168 len 20480 data: 11 14 11 10 9 27/30 Write block 558 len 28672 data: 12 12 15 14 16 16 15 28/30 Read block 406 len 4096 29/30 Write block 358 len 32768 data: 11 11 12 8 9 9 6 6 30/30 Read block 14 len 8192 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Sent Sent Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 4 4 6 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:44.206474279Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1708} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 126 126 126 207 207 207 F F F 3 060-079 130 130 130 214 214 214 F F F 4 080-099 130 130 130 214 214 214 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 127 127 127 208 208 208 T T T 7 140-159 130 130 130 212 212 212 T T T 8 160-179 130 130 130 214 214 214 T T T 9 180-199 129 129 129 211 211 211 F F F 10 200-219 129 129 129 211 211 211 F F F 11 220-239 130 130 130 213 213 213 F F F 12 240-259 130 130 130 214 214 214 F F F 13 260-279 130 130 130 214 214 214 F F F 14 280-299 128 128 128 209 209 209 F F F 15 300-319 126 126 126 207 207 207 F F F 16 320-339 130 130 130 214 214 214 F F F 17 340-359 127 127 127 208 208 208 T T T 18 360-379 126 126 126 207 207 207 T T T 19 380-399 118 118 118 193 193 193 F F F 20 400-419 130 130 130 214 214 214 F F F 21 420-439 128 128 128 209 209 209 F F F 22 440-459 129 129 129 211 211 211 F F F 23 460-479 130 130 130 213 213 213 F F F 24 480-499 130 130 130 213 213 213 F F F 25 500-519 125 125 125 206 206 206 F F F 26 520-539 127 127 127 208 208 208 F F F 27 540-559 125 125 125 206 206 206 T T T 28 560-579 127 127 127 208 208 208 T T T 29 580-599 128 128 128 209 209 209 F F F Max gen: 130, Max flush: 214 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 126 126 126 207 207 207 F F F 3 060-079 130 130 130 214 214 214 F F F 4 080-099 130 130 130 214 214 214 F F F 5 100-119 118 118 118 192 192 192 F F F 6 120-139 131 131 131 215 215 215 F F F 7 140-159 131 131 131 215 215 215 F F F 8 160-179 131 131 131 215 215 215 F F F 9 180-199 129 129 129 211 211 211 F F F 10 200-219 129 129 129 211 211 211 F F F 11 220-239 130 130 130 213 213 213 F F F 12 240-259 130 130 130 214 214 214 F F F 13 260-279 130 130 130 214 214 214 F F F 14 280-299 128 128 128 209 209 209 F F F 15 300-319 126 126 126 207 207 207 F F F 16 320-339 130 130 130 214 214 214 F F F 17 340-359 131 131 131 215 215 215 F F F 18 360-379 131 131 131 215 215 215 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 130 130 130 214 214 214 F F F 21 420-439 128 128 128 209 209 209 F F F 22 440-459 129 129 129 211 211 211 F F F 23 460-479 130 130 130 213 213 213 F F F 24 480-499 130 130 130 213 213 213 F F F 25 500-519 125 125 125 206 206 206 F F F 26 520-539 127 127 127 208 208 208 F F F 27 540-559 131 131 131 215 215 215 F F F 28 560-579 131 131 131 215 215 215 F F F 29 580-599 128 128 128 209 209 209 F F F Max gen: 131, Max flush: 215 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.461550677Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.506715762Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.507472166Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.507514225Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.507529688Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.507544941Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716} {"msg":"Crucible 6775d750-6086-4517-b74d-ea2ed458e8c3 has session id: 3c610566-e9d3-4072-bbfc-76054f7ac6d2","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508061133Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Upstairs opts: Upstairs UUID: 6775d750-6086-4517-b74d-ea2ed458e8c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508105882Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Crucible stats registered with UUID: 6775d750-6086-4517-b74d-ea2ed458e8c3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508122126Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:48.508134328Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508622254Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.50865125Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716} {"msg":"6775d750-6086-4517-b74d-ea2ed458e8c3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508710743Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508728417Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"0","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508757943Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"0","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508782579Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"1","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508810385Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"1","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.50883672Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"2","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508865946Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"2","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508891022Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"io task","client":"2","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508919488Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"io task","client":"0","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.508945674Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"io task","client":"1","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43485)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.50897559Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"io task","client":"2","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:51289)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.509003786Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"io task","client":"0","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:54827)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.509034923Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"io task","client":"1","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.509588542Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"2","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.509620169Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"0","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.510238793Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"0","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.510395527Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"2","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.510793272Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.510881111Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"1","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.51092307Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.51138595Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"1","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"[0]R flush_numbers[0..12]: [198, 202, 207, 214, 214, 192, 215, 215, 215, 211, 211, 213]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.511910043Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"[0]R generation[0..12]: [122, 124, 126, 130, 130, 118, 131, 131, 131, 129, 129, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.511934879Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.511950922Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"[1]R flush_numbers[0..12]: [198, 202, 207, 214, 214, 192, 215, 215, 215, 211, 211, 213]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.511965265Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"[1]R generation[0..12]: [122, 124, 126, 130, 130, 118, 131, 131, 131, 129, 129, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.511978698Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.511992241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"[2]R flush_numbers[0..12]: [198, 202, 207, 214, 214, 192, 215, 215, 215, 211, 211, 213]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512005524Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"[2]R generation[0..12]: [122, 124, 126, 130, 130, 118, 131, 131, 131, 129, 129, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512022707Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512038481Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512051164Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512063836Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512076459Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512088782Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"":"downstairs","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:27:48.512102545Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512117948Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"0","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512131171Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"1","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512144074Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"client":"2","":"downstairs","session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512156907Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512168789Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"6775d750-6086-4517-b74d-ea2ed458e8c3 is now active with session: 3c610566-e9d3-4072-bbfc-76054f7ac6d2","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512181122Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512192604Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716,"session_id":"3c610566-e9d3-4072-bbfc-76054f7ac6d2"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512204357Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716} {"msg":"Activated sub_volume 6775d750-6086-4517-b74d-ea2ed458e8c3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:27:48.512215099Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 287 len 24576 data: 10 10 8 11 9 9 02/30 Write block 365 len 28672 data: 7 5 6 5 4 6 6 03/30 Read block 575 len 4096 04/30 Write block 247 len 12288 data: 10 9 10 05/30 Write block 306 len 24576 data: 11 11 12 11 8 9 06/30 Write block 74 len 16384 data: 17 15 13 11 07/30 Write block 233 len 12288 data: 13 12 14 08/30 Read block 546 len 32768 09/30 Flush 10/30 Write block 424 len 20480 data: 8 6 9 11 13 11/30 Flush 12/30 Write block 170 len 28672 data: 12 11 10 8 7 7 10 13/30 Read block 395 len 28672 14/30 Write block 593 len 24576 data: 10 8 6 7 6 6 15/30 Write block 543 len 8192 data: 14 15 16/30 Read block 487 len 28672 17/30 Read block 16 len 8192 18/30 Read block 240 len 12288 19/30 Read block 555 len 12288 20/30 Write block 572 len 20480 data: 10 11 9 10 10 21/30 Write block 71 len 40960 data: 13 14 13 18 16 14 12 11 6 7 22/30 Write block 181 len 16384 data: 13 13 13 16 23/30 Write block 283 len 40960 data: 12 12 14 13 11 11 9 12 10 10 24/30 Write block 127 len 40960 data: 6 7 7 7 8 9 8 8 8 8 25/30 Read block 512 len 12288 26/30 Write block 119 len 4096 data: 13 27/30 Write block 480 len 4096 data: 9 28/30 Write block 429 len 12288 data: 12 13 13 29/30 Read block 83 len 28672 30/30 Write block 473 len 20480 data: 11 11 12 8 8 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 7 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:27:48.530326947Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1716} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 126 126 126 207 207 207 F F F 3 060-079 132 132 132 216 216 216 T T T 4 080-099 130 130 130 214 214 214 T T T 5 100-119 118 118 118 192 192 192 T T T 6 120-139 131 131 131 215 215 215 T T T 7 140-159 131 131 131 215 215 215 F F F 8 160-179 131 131 131 215 215 215 T T T 9 180-199 129 129 129 211 211 211 T T T 10 200-219 129 129 129 211 211 211 F F F 11 220-239 132 132 132 216 216 216 F F F 12 240-259 132 132 132 216 216 216 F F F 13 260-279 130 130 130 214 214 214 F F F 14 280-299 132 132 132 216 216 216 T T T 15 300-319 132 132 132 216 216 216 F F F 16 320-339 130 130 130 214 214 214 F F F 17 340-359 131 131 131 215 215 215 F F F 18 360-379 132 132 132 216 216 216 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 130 130 130 214 214 214 F F F 21 420-439 132 132 132 217 217 217 T T T 22 440-459 129 129 129 211 211 211 F F F 23 460-479 130 130 130 213 213 213 T T T 24 480-499 130 130 130 213 213 213 T T T 25 500-519 125 125 125 206 206 206 F F F 26 520-539 127 127 127 208 208 208 F F F 27 540-559 131 131 131 215 215 215 T T T 28 560-579 131 131 131 215 215 215 T T T 29 580-599 128 128 128 209 209 209 T T T Max gen: 132, Max flush: 217 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 126 126 126 207 207 207 F F F 3 060-079 133 133 133 218 218 218 F F F 4 080-099 133 133 133 218 218 218 F F F 5 100-119 133 133 133 218 218 218 F F F 6 120-139 133 133 133 218 218 218 F F F 7 140-159 131 131 131 215 215 215 F F F 8 160-179 133 133 133 218 218 218 F F F 9 180-199 133 133 133 218 218 218 F F F 10 200-219 129 129 129 211 211 211 F F F 11 220-239 132 132 132 216 216 216 F F F 12 240-259 132 132 132 216 216 216 F F F 13 260-279 130 130 130 214 214 214 F F F 14 280-299 133 133 133 218 218 218 F F F 15 300-319 132 132 132 216 216 216 F F F 16 320-339 130 130 130 214 214 214 F F F 17 340-359 131 131 131 215 215 215 F F F 18 360-379 132 132 132 216 216 216 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 130 130 130 214 214 214 F F F 21 420-439 133 133 133 218 218 218 F F F 22 440-459 129 129 129 211 211 211 F F F 23 460-479 133 133 133 218 218 218 F F F 24 480-499 133 133 133 218 218 218 F F F 25 500-519 125 125 125 206 206 206 F F F 26 520-539 127 127 127 208 208 208 F F F 27 540-559 133 133 133 218 218 218 F F F 28 560-579 133 133 133 218 218 218 F F F 29 580-599 133 133 133 218 218 218 F F F Max gen: 133, Max flush: 218 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.818421702Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:02.861432152Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.861858824Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.893023509Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.893617727Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.893660836Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.89367726Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.893694683Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724} {"msg":"Crucible f7c891ba-d872-4789-a296-f0cc00615299 has session id: fabbd994-f5e8-4aae-8337-1c4f89583c0a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.89423776Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Upstairs opts: Upstairs UUID: f7c891ba-d872-4789-a296-f0cc00615299, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.894334641Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Crucible stats registered with UUID: f7c891ba-d872-4789-a296-f0cc00615299","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.894364468Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:02.894385362Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.894849063Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.894876358Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724} {"msg":"f7c891ba-d872-4789-a296-f0cc00615299 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.894946304Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.894964487Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"0","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.894992643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"0","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.895006966Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"1","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.895025101Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"1","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.895038893Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"2","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.895053717Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"2","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.895067039Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.895080722Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.895096856Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:65393)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.895281646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53507)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.895499603Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62480)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.895545653Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.89595147Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"2","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.896115196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"1","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.89631922Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"0","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.896611133Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"1","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.896844603Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"2","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897027762Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"0","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897209482Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897419507Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"[0]R flush_numbers[0..12]: [198, 202, 207, 218, 218, 218, 218, 215, 218, 218, 211, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897534282Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"[0]R generation[0..12]: [122, 124, 126, 133, 133, 133, 133, 131, 133, 133, 129, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897555416Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.89757075Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"[1]R flush_numbers[0..12]: [198, 202, 207, 218, 218, 218, 218, 215, 218, 218, 211, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897584473Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"[1]R generation[0..12]: [122, 124, 126, 133, 133, 133, 133, 131, 133, 133, 129, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897598316Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897612359Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"[2]R flush_numbers[0..12]: [198, 202, 207, 218, 218, 218, 218, 215, 218, 218, 211, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897625822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"[2]R generation[0..12]: [122, 124, 126, 133, 133, 133, 133, 131, 133, 133, 129, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897639025Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897652507Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.89766481Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897677713Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897694336Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":"downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897707699Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"":Wait for a query_work_queue command to finish before sending IO "downstairs","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897727193Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897738806Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"0","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897751789Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"1","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897764621Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"client":"2","":"downstairs","session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897777494Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897789437Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"f7c891ba-d872-4789-a296-f0cc00615299 is now active with session: fabbd994-f5e8-4aae-8337-1c4f89583c0a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.89780182Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897813442Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724,"session_id":"fabbd994-f5e8-4aae-8337-1c4f89583c0a"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897825395Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724} {"msg":"Activated sub_volume f7c891ba-d872-4789-a296-f0cc00615299","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:02.897836417Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 445 len 4096 data: 12 02/30 Write block 371 len 36864 data: 7 6 7 7 10 7 6 8 10 03/30 Write block 352 len 8192 data: 6 7 04/30 Read block 46 len 40960 05/30 Read block 234 len 12288 06/30 Flush 07/30 Write block 487 len 16384 data: 10 11 15 15 08/30 Read block 156 len 24576 09/30 Read block 9 len 28672 10/30 Write block 357 len 36864 data: 9 12 12 13 9 10 10 7 8 11/30 Read block 42 len 36864 12/30 Flush 13/30 Write block 287 len 28672 data: 12 12 10 13 11 11 8 14/30 Flush 15/30 Write block 355 len 4096 data: 7 16/30 Read block 241 len 16384 17/30 Write block 580 len 20480 data: 8 9 9 8 11 18/30 Flush 19/30 Write block 574 len 12288 data: 10 11 11 20/30 Write block 593 len 20480 data: 11 9 7 8 7 21/30 Read block 223 len 24576 22/30 Write block 272 len 36864 data: 11 11 10 9 11 8 9 8 11 23/30 Read block 26 len 24576 24/30 Read block 410 len 40960 25/30 Read block 505 len 20480 26/30 Read block 90 len 28672 27/30 Write block 499 len 28672 data: 12 12 14 13 13 11 10 28/30 Read block 8 len 8192 29/30 Write block 263 len 28672 data: 12 13 15 13 12 8 8 30/30 Write block 340 len 24576 data: 11 9 12 10 11 9 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 3 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:02.918811433Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1724} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 126 126 126 207 207 207 F F F 3 060-079 133 133 133 218 218 218 F F F 4 080-099 133 133 133 218 218 218 F F F 5 100-119 133 133 133 218 218 218 F F F 6 120-139 133 133 133 218 218 218 F F F 7 140-159 131 131 131 215 215 215 F F F 8 160-179 133 133 133 218 218 218 F F F 9 180-199 133 133 133 218 218 218 F F F 10 200-219 129 129 129 211 211 211 F F F 11 220-239 132 132 132 216 216 216 F F F 12 240-259 132 132 132 216 216 216 F F F 13 260-279 130 130 130 214 214 214 T T T 14 280-299 134 134 134 221 221 221 T T T 15 300-319 132 132 132 216 216 216 F F F 16 320-339 130 130 130 214 214 214 F F F 17 340-359 134 134 134 222 222 222 T T T 18 360-379 134 134 134 220 220 220 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 130 130 130 214 214 214 F F F 21 420-439 133 133 133 218 218 218 F F F 22 440-459 134 134 134 219 219 219 F F F 23 460-479 133 133 133 218 218 218 F F F 24 480-499 134 134 134 220 220 220 T T T 25 500-519 125 125 125 206 206 206 T T T 26 520-539 127 127 127 208 208 208 F F F 27 540-559 133 133 133 218 218 218 F F F 28 560-579 133 133 133 218 218 218 T T T 29 580-599 134 134 134 222 222 222 T T T Max gen: 134, Max flush: 222 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 126 126 126 207 207 207 F F F 3 060-079 133 133 133 218 218 218 F F F 4 080-099 133 133 133 218 218 218 F F F 5 100-119 133 133 133 218 218 218 F F F 6 120-139 133 133 133 218 218 218 F F F 7 140-159 131 131 131 215 215 215 F F F 8 160-179 133 133 133 218 218 218 F F F 9 180-199 133 133 133 218 218 218 F F F 10 200-219 129 129 129 211 211 211 F F F 11 220-239 132 132 132 216 216 216 F F F 12 240-259 132 132 132 216 216 216 F F F 13 260-279 135 135 135 223 223 223 F F F 14 280-299 135 135 135 223 223 223 F F F 15 300-319 132 132 132 216 216 216 F F F 16 320-339 130 130 130 214 214 214 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 134 134 134 220 220 220 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 130 130 130 214 214 214 F F F 21 420-439 133 133 133 218 218 218 F F F 22 440-459 134 134 134 219 219 219 F F F 23 460-479 133 133 133 218 218 218 F F F 24 480-499 135 135 135 223 223 223 F F F 25 500-519 135 135 135 223 223 223 F F F 26 520-539 127 127 127 208 208 208 F F F 27 540-559 133 133 133 218 218 218 F F F 28 560-579 135 135 135 223 223 223 F F F 29 580-599 135 135 135 223 223 223 F F F Max gen: 135, Max flush: 223 Begin loop 68 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.199011672Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.242479191Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.243155217Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.243199246Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.24321447Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.243228913Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732} {"msg":"Crucible dafdde21-6d56-4b29-92d6-24f0a0185895 has session id: 7a3c2944-26ab-4788-9d4c-95d9f64c139e","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.243933365Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Upstairs opts: Upstairs UUID: dafdde21-6d56-4b29-92d6-24f0a0185895, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.243976914Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Crucible stats registered with UUID: dafdde21-6d56-4b29-92d6-24f0a0185895","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244024094Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:07.244039838Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244499247Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244525393Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732} {"msg":"dafdde21-6d56-4b29-92d6-24f0a0185895 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244603089Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244621513Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"0","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244649239Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"0","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244662912Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"1","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244676755Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"1","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244689698Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"2","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244703501Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"2","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244716284Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244729657Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"io task","client":"0","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.24474302Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"io task","client":"1","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50099)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244833719Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61663)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244900703Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"io task","client":"1","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54751)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.244947804Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"io task","client":"0","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.245508085Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"0","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.24553618Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"2","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.246157174Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"2","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.246195333Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"0","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.246838261Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.246927241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.247131245Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"1","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.247689385Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"1","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"[0]R flush_numbers[0..12]: [198, 202, 207, 218, 218, 218, 218, 215, 218, 218, 211, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248253057Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"[0]R generation[0..12]: [122, 124, 126, 133, 133, 133, 133, 131, 133, 133, 129, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248277942Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248292555Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"[1]R flush_numbers[0..12]: [198, 202, 207, 218, 218, 218, 218, 215, 218, 218, 211, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248306218Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"[1]R generation[0..12]: [122, 124, 126, 133, 133, 133, 133, 131, 133, 133, 129, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248319881Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248337505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"[2]R flush_numbers[0..12]: [198, 202, 207, 218, 218, 218, 218, 215, 218, 218, 211, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248351538Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"[2]R generation[0..12]: [122, 124, 126, 133, 133, 133, 133, 131, 133, 133, 129, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248364931Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248378644Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248391317Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.24840437Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248417072Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":"downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248429645Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"":Wait for a query_work_queue command to finish before sending IO "downstairs","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248449769Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248461442Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"0","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248474285Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"1","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248486787Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"client":"2","":"downstairs","session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.2484995Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248511462Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"dafdde21-6d56-4b29-92d6-24f0a0185895 is now active with session: 7a3c2944-26ab-4788-9d4c-95d9f64c139e","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248523755Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248537158Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732,"session_id":"7a3c2944-26ab-4788-9d4c-95d9f64c139e"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248549431Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732} {"msg":"Activated sub_volume dafdde21-6d56-4b29-92d6-24f0a0185895","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:07.248560313Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 578 len 32768 data: 13 13 9 10 10 9 12 13 02/30 Read block 361 len 16384 03/30 Read block 115 len 16384 04/30 Read block 2 len 24576 05/30 Flush 06/30 Write block 57 len 8192 data: 10 12 07/30 Write block 88 len 20480 data: 9 8 10 10 9 08/30 Write block 133 len 36864 data: 9 9 9 9 7 10 12 11 10 09/30 Read block 567 len 20480 10/30 Write block 536 len 16384 data: 8 10 10 12 11/30 Write block 319 len 32768 data: 8 10 10 10 11 11 13 12 12/30 Read block 194 len 8192 13/30 Read block 159 len 12288 14/30 Flush 15/30 Write block 251 len 32768 data: 11 10 10 11 10 9 10 11 16/30 Read block 155 len 36864 17/30 Flush 18/30 Write block 587 len 40960 data: 10 10 12 11 12 10 12 10 8 9 19/30 Write block 469 len 16384 data: 12 10 9 10 20/30 Read block 469 len 32768 21/30 Read block 49 len 40960 22/30 Flush 23/30 Flush 24/30 Write block 400 len 40960 data: 10 9 10 11 11 12 13 15 13 12 25/30 Read block 236 len 40960 26/30 Write block 198 len 40960 data: 11 10 14 16 13 11 10 13 12 10 27/30 Read block 458 len 12288 28/30 Write block 1 len 12288 data: 6 8 9 29/30 Read block 173 len 32768 30/30 Write block 490 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 8 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:07.269744744Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1732} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 198 198 198 T T T 1 020-039 124 124 124 202 202 202 F F F 2 040-059 136 136 136 225 225 225 F F F 3 060-079 133 133 133 218 218 218 F F F 4 080-099 136 136 136 225 225 225 F F F 5 100-119 133 133 133 218 218 218 F F F 6 120-139 136 136 136 225 225 225 F F F 7 140-159 136 136 136 225 225 225 F F F 8 160-179 133 133 133 218 218 218 F F F 9 180-199 133 133 133 218 218 218 T T T 10 200-219 129 129 129 211 211 211 T T T 11 220-239 132 132 132 216 216 216 F F F 12 240-259 136 136 136 226 226 226 F F F 13 260-279 135 135 135 223 223 223 F F F 14 280-299 135 135 135 223 223 223 F F F 15 300-319 136 136 136 225 225 225 F F F 16 320-339 136 136 136 225 225 225 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 134 134 134 220 220 220 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 130 130 130 214 214 214 T T T 21 420-439 133 133 133 218 218 218 F F F 22 440-459 134 134 134 219 219 219 F F F 23 460-479 136 136 136 227 227 227 F F F 24 480-499 135 135 135 223 223 223 T T T 25 500-519 135 135 135 223 223 223 F F F 26 520-539 136 136 136 225 225 225 F F F 27 540-559 133 133 133 218 218 218 F F F 28 560-579 136 136 136 224 224 224 F F F 29 580-599 136 136 136 227 227 227 F F F Max gen: 136, Max flush: 227 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 228 228 228 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 136 136 136 225 225 225 F F F 3 060-079 133 133 133 218 218 218 F F F 4 080-099 136 136 136 225 225 225 F F F 5 100-119 133 133 133 218 218 218 F F F 6 120-139 136 136 136 225 225 225 F F F 7 140-159 136 136 136 225 225 225 F F F 8 160-179 133 133 133 218 218 218 F F F 9 180-199 137 137 137 228 228 228 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 132 132 132 216 216 216 F F F 12 240-259 136 136 136 226 226 226 F F F 13 260-279 135 135 135 223 223 223 F F F 14 280-299 135 135 135 223 223 223 F F F 15 300-319 136 136 136 225 225 225 F F F 16 320-339 136 136 136 225 225 225 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 134 134 134 220 220 220 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 137 137 137 228 228 228 F F F 21 420-439 133 133 133 218 218 218 F F F 22 440-459 134 134 134 219 219 219 F F F 23 460-479 136 136 136 227 227 227 F F F 24 480-499 137 137 137 228 228 228 F F F 25 500-519 135 135 135 223 223 223 F F F 26 520-539 136 136 136 225 225 225 F F F 27 540-559 133 133 133 218 218 218 F F F 28 560-579 136 136 136 224 224 224 F F F 29 580-599 136 136 136 227 227 227 F F F Max gen: 137, Max flush: 228 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.507191438Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.551602501Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.552382769Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.552427349Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.552446293Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.552461436Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740} {"msg":"Crucible ef5f9869-6056-4d57-91cd-e2e6248aacbe has session id: 27d7b74e-11d1-47e6-b5a0-180b5623fdbd","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.5530839Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Upstairs opts: Upstairs UUID: ef5f9869-6056-4d57-91cd-e2e6248aacbe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553118608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Crucible stats registered with UUID: ef5f9869-6056-4d57-91cd-e2e6248aacbe","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553134611Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:11.553148044Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553614495Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553640631Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740} {"msg":"ef5f9869-6056-4d57-91cd-e2e6248aacbe active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553704054Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.55372934Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"0","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553757366Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"0","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553771149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"1","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553784952Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"1","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553797805Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"2","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553819249Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"2","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553841714Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"io task","client":"2","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553866619Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"io task","client":"0","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553890054Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:45973)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.553977383Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"io task","client":"0","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:11.554023723Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:38684)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.554059671Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"io task","client":"2","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:11.554093268Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:11.554125705Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.55414999Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.554630234Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"2","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.554664101Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"0","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.55535427Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"0","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.555380716Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"2","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.555971163Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:11.55604873Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.556063515Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:52923)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.556243974Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.557255442Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"1","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.55775678Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"1","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"[0]R flush_numbers[0..12]: [228, 202, 225, 218, 225, 218, 225, 225, 218, 228, 228, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558363971Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"[0]R generation[0..12]: [137, 124, 136, 133, 136, 133, 136, 136, 133, 137, 137, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558394227Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558413511Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"[1]R flush_numbers[0..12]: [228, 202, 225, 218, 225, 218, 225, 225, 218, 228, 228, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558430665Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"[1]R generation[0..12]: [137, 124, 136, 133, 136, 133, 136, 136, 133, 137, 137, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558447279Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558464313Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"[2]R flush_numbers[0..12]: [228, 202, 225, 218, 225, 218, 225, 225, 218, 228, 228, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558481056Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"[2]R generation[0..12]: [137, 124, 136, 133, 136, 133, 136, 136, 133, 137, 137, 132]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.55849786Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558520815Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558537218Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558553102Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558568895Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:28:21.558584519Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"":"downstairs","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558604783Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558619166Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"0","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558635199Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"1","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558650823Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"client":"2","":"downstairs","session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558666716Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558681299Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"ef5f9869-6056-4d57-91cd-e2e6248aacbe is now active with session: 27d7b74e-11d1-47e6-b5a0-180b5623fdbd","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558696643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558710986Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740,"session_id":"27d7b74e-11d1-47e6-b5a0-180b5623fdbd"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558725719Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740} {"msg":"Activated sub_volume ef5f9869-6056-4d57-91cd-e2e6248aacbe","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:21.558739112Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 85 len 4096 data: 8 02/30 Read block 183 len 16384 03/30 Flush 04/30 Write block 128 len 24576 data: 8 8 8 9 10 10 05/30 Read block 439 len 36864 06/30 Read block 28 len 36864 07/30 Read block 122 len 32768 08/30 Write block 408 len 4096 data: 14 09/30 Read block 554 len 4096 10/30 Read block 402 len 36864 11/30 Write block 252 len 4096 data: 11 12/30 Write block 311 len 32768 data: 10 9 10 9 9 11 10 7 13/30 Read block 218 len 24576 14/30 Read block 84 len 28672 15/30 Write block 93 len 32768 data: 8 6 6 6 9 9 9 6 16/30 Write block 60 len 32768 data: 8 8 9 9 10 9 10 11 17/30 Flush 18/30 Write block 407 len 8192 data: 16 15 19/30 Write block 119 len 4096 data: 14 20/30 Read block 483 len 24576 21/30 Flush 22/30 Write block 171 len 16384 data: 12 11 9 8 23/30 Write block 226 len 4096 data: 8 24/30 Read block 17 len 24576 25/30 Read block 460 len 40960 26/30 Read block 468 len 28672 27/30 Write block 149 len 24576 data: 13 11 15 14 13 16 28/30 Read block 336 len 16384 29/30 Read block 498 len 24576 30/30 Read block 26 len 24576 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:21.57847271Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1740} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 228 228 228 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 136 136 136 225 225 225 F F F 3 060-079 138 138 138 230 230 230 F F F 4 080-099 138 138 138 230 230 230 F F F 5 100-119 138 138 138 231 231 231 F F F 6 120-139 138 138 138 230 230 230 F F F 7 140-159 136 136 136 225 225 225 T T T 8 160-179 133 133 133 218 218 218 T T T 9 180-199 137 137 137 228 228 228 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 132 132 132 216 216 216 T T T 12 240-259 138 138 138 230 230 230 F F F 13 260-279 135 135 135 223 223 223 F F F 14 280-299 135 135 135 223 223 223 F F F 15 300-319 138 138 138 230 230 230 F F F 16 320-339 136 136 136 225 225 225 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 134 134 134 220 220 220 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 138 138 138 231 231 231 F F F 21 420-439 133 133 133 218 218 218 F F F 22 440-459 134 134 134 219 219 219 F F F 23 460-479 136 136 136 227 227 227 F F F 24 480-499 137 137 137 228 228 228 F F F 25 500-519 135 135 135 223 223 223 F F F 26 520-539 136 136 136 225 225 225 F F F 27 540-559 133 133 133 218 218 218 F F F 28 560-579 136 136 136 224 224 224 F F F 29 580-599 136 136 136 227 227 227 F F F Max gen: 138, Max flush: 231 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 228 228 228 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 136 136 136 225 225 225 F F F 3 060-079 138 138 138 230 230 230 F F F 4 080-099 138 138 138 230 230 230 F F F 5 100-119 138 138 138 231 231 231 F F F 6 120-139 138 138 138 230 230 230 F F F 7 140-159 139 139 139 232 232 232 F F F 8 160-179 139 139 139 232 232 232 F F F 9 180-199 137 137 137 228 228 228 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 139 139 139 232 232 232 F F F 12 240-259 138 138 138 230 230 230 F F F 13 260-279 135 135 135 223 223 223 F F F 14 280-299 135 135 135 223 223 223 F F F 15 300-319 138 138 138 230 230 230 F F F 16 320-339 136 136 136 225 225 225 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 134 134 134 220 220 220 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 138 138 138 231 231 231 F F F 21 420-439 133 133 133 218 218 218 F F F 22 440-459 134 134 134 219 219 219 F F F 23 460-479 136 136 136 227 227 227 F F F 24 480-499 137 137 137 228 228 228 F F F 25 500-519 135 135 135 223 223 223 F F F 26 520-539 136 136 136 225 225 225 F F F 27 540-559 133 133 133 218 218 218 F F F 28 560-579 136 136 136 224 224 224 F F F 29 580-599 136 136 136 227 227 227 F F F Max gen: 139, Max flush: 232 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.808979735Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.852447174Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.853099005Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.853140624Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.853155127Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.85316921Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748} {"msg":"Crucible 74371846-5392-4b5a-a0d0-3d094ed382d8 has session id: 92cdf521-ffcd-4a5a-ba22-288d0e1086ba","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.853746945Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Upstairs opts: Upstairs UUID: 74371846-5392-4b5a-a0d0-3d094ed382d8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.853797946Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Crucible stats registered with UUID: 74371846-5392-4b5a-a0d0-3d094ed382d8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.853824832Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:25.853846246Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854179758Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854202753Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748} {"msg":"74371846-5392-4b5a-a0d0-3d094ed382d8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854275809Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854298904Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"0","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854341593Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"0","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854368739Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"1","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854396565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"1","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854423041Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"2","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854448866Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"2","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854473432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854498137Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854529574Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:62644)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.85455676Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:45325)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.854581995Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:25.854612602Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:25.854689938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:25.854723356Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.85474206Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.855135534Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"2","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.855163551Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"0","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.855812951Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"0","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.855837866Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"2","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.856423342Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:25.856514362Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.855820254Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:45959)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.856023687Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.856999918Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"1","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.857552117Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"1","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"[0]R flush_numbers[0..12]: [228, 202, 225, 230, 230, 231, 230, 232, 232, 228, 228, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858151657Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"[0]R generation[0..12]: [137, 124, 136, 138, 138, 138, 138, 139, 139, 137, 137, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858173471Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858188665Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"[1]R flush_numbers[0..12]: [228, 202, 225, 230, 230, 231, 230, 232, 232, 228, 228, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858202297Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"[1]R generation[0..12]: [137, 124, 136, 138, 138, 138, 138, 139, 139, 137, 137, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.85821582Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858229733Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"[2]R flush_numbers[0..12]: [228, 202, 225, 230, 230, 231, 230, 232, 232, 228, 228, 232]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858243296Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"[2]R generation[0..12]: [137, 124, 136, 138, 138, 138, 138, 139, 139, 137, 137, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858256709Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858270472Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858283045Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858295737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858311741Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858324734Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"":"downstairs","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858338487Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858350149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"0","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858362982Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"1","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858375535Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"client":"2","":"downstairs","session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:28:35.858388228Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858404031Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"74371846-5392-4b5a-a0d0-3d094ed382d8 is now active with session: 92cdf521-ffcd-4a5a-ba22-288d0e1086ba","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858416394Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858427956Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748,"session_id":"92cdf521-ffcd-4a5a-ba22-288d0e1086ba"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858439669Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748} {"msg":"Activated sub_volume 74371846-5392-4b5a-a0d0-3d094ed382d8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:35.858450361Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 55 len 32768 data: 12 10 11 13 10 9 9 10 02/30 Read block 508 len 24576 03/30 Write block 289 len 36864 data: 11 14 12 12 9 9 8 12 13 04/30 Read block 184 len 20480 05/30 Read block 421 len 32768 06/30 Flush 07/30 Write block 236 len 24576 data: 11 10 10 9 10 9 08/30 Write block 543 len 32768 data: 15 16 14 16 13 12 12 11 09/30 Read block 590 len 4096 10/30 Read block 91 len 8192 11/30 Write block 418 len 24576 data: 7 7 6 4 5 7 12/30 Write block 482 len 4096 data: 11 13/30 Read block 40 len 28672 14/30 Read block 254 len 36864 15/30 Write block 438 len 24576 data: 7 10 13 10 13 13 16/30 Flush 17/30 Write block 64 len 8192 data: 11 10 18/30 Write block 570 len 8192 data: 11 12 19/30 Read block 216 len 32768 20/30 Read block 592 len 8192 21/30 Write block 464 len 8192 data: 11 10 22/30 Write block 129 len 4096 data: 9 23/30 Read block 534 len 24576 24/30 Write block 302 len 40960 data: 13 13 13 11 12 12 13 12 9 11 25/30 Read block 455 len 28672 26/30 Read block 46 len 28672 27/30 Write block 77 len 36864 data: 13 12 7 8 5 4 6 6 9 28/30 Write block 184 len 28672 data: 17 15 11 10 10 9 7 29/30 Write block 317 len 4096 data: 11 30/30 Write block 565 len 20480 data: 14 13 15 14 14 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 2 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 9 Sent Done Sent false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 3 4 11 Done 10 11 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:35.878589756Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1748} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 228 228 228 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 140 140 140 233 233 233 F F F 3 060-079 140 140 140 233 233 233 T T T 4 080-099 138 138 138 230 230 230 T T T 5 100-119 138 138 138 231 231 231 F F F 6 120-139 138 138 138 230 230 230 T T T 7 140-159 139 139 139 232 232 232 F F F 8 160-179 139 139 139 232 232 232 F F F 9 180-199 137 137 137 228 228 228 T T T 10 200-219 137 137 137 228 228 228 F F F 11 220-239 140 140 140 234 234 234 F F F 12 240-259 140 140 140 234 234 234 F F F 13 260-279 135 135 135 223 223 223 F F F 14 280-299 140 140 140 233 233 233 F F F 15 300-319 138 138 138 230 230 230 T T T 16 320-339 136 136 136 225 225 225 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 134 134 134 220 220 220 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 140 140 140 234 234 234 F F F 21 420-439 140 140 140 234 234 234 F F F 22 440-459 140 140 140 234 234 234 F F F 23 460-479 136 136 136 227 227 227 T T T 24 480-499 140 140 140 234 234 234 F F F 25 500-519 135 135 135 223 223 223 F F F 26 520-539 136 136 136 225 225 225 F F F 27 540-559 140 140 140 234 234 234 F F F 28 560-579 136 136 136 224 224 224 T T T 29 580-599 136 136 136 227 227 227 F F F Max gen: 140, Max flush: 234 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 228 228 228 F F F 1 020-039 124 124 124 202 202 202 F F F 2 040-059 140 140 140 233 233 233 F F F 3 060-079 141 141 141 235 235 235 F F F 4 080-099 141 141 141 235 235 235 F F F 5 100-119 138 138 138 231 231 231 F F F 6 120-139 141 141 141 235 235 235 F F F 7 140-159 139 139 139 232 232 232 F F F 8 160-179 139 139 139 232 232 232 F F F 9 180-199 141 141 141 235 235 235 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 140 140 140 234 234 234 F F F 12 240-259 140 140 140 234 234 234 F F F 13 260-279 135 135 135 223 223 223 F F F 14 280-299 140 140 140 233 233 233 F F F 15 300-319 141 141 141 235 235 235 F F F 16 320-339 136 136 136 225 225 225 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 134 134 134 220 220 220 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 140 140 140 234 234 234 F F F 21 420-439 140 140 140 234 234 234 F F F 22 440-459 140 140 140 234 234 234 F F F 23 460-479 141 141 141 235 235 235 F F F 24 480-499 140 140 140 234 234 234 F F F 25 500-519 135 135 135 223 223 223 F F F 26 520-539 136 136 136 225 225 225 F F F 27 540-559 140 140 140 234 234 234 F F F 28 560-579 141 141 141 235 235 235 F F F 29 580-599 136 136 136 227 227 227 F F F Max gen: 141, Max flush: 235 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.145288552Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.190167325Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.191021169Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.191066149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.191081833Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.191096516Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756} {"msg":"Crucible 432ead01-768f-4cd9-bcd9-6361d8646ddf has session id: 4e9c613a-cd76-4b53-833b-cac88773cf53","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.191755808Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Upstairs opts: Upstairs UUID: 432ead01-768f-4cd9-bcd9-6361d8646ddf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.191798777Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Crucible stats registered with UUID: 432ead01-768f-4cd9-bcd9-6361d8646ddf","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.191825933Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:40.191839926Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192405608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192432824Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756} {"msg":"432ead01-768f-4cd9-bcd9-6361d8646ddf active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192480504Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192495808Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"0","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192525244Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"0","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192538807Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"1","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.19255276Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"1","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192565753Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"2","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192579836Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"2","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192592969Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192606672Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192620175Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"ds_connection connected from Ok(127.0.0.1:40875)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192779749Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"ds_connection connected from Ok(127.0.0.1:52363)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192824469Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"ds_connection connected from Ok(127.0.0.1:57636)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.192849634Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.1935248Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"2","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.193553366Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"0","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.194131371Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"0","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.194258828Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"2","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.194702054Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.194810107Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.195016262Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"1","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.19556314Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"1","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"[0]R flush_numbers[0..12]: [228, 202, 233, 235, 235, 231, 235, 232, 232, 235, 228, 234]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196135213Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"[0]R generation[0..12]: [137, 124, 140, 141, 141, 138, 141, 139, 139, 141, 137, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196155257Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196169711Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"[1]R flush_numbers[0..12]: [228, 202, 233, 235, 235, 231, 235, 232, 232, 235, 228, 234]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196183274Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"[1]R generation[0..12]: [137, 124, 140, 141, 141, 138, 141, 139, 139, 141, 137, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196199617Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.19621344Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"[2]R flush_numbers[0..12]: [228, 202, 233, 235, 235, 231, 235, 232, 232, 235, 228, 234]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196226873Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"[2]R generation[0..12]: [137, 124, 140, 141, 141, 138, 141, 139, 139, 141, 137, 140]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196240116Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196253789Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196266161Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196278724Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196292247Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196311621Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"":"downstairs","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196324494Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196336066Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"0","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196348689Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"1","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196361272Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"client":"2","":"downstairs","session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196373795Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196385577Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"432ead01-768f-4cd9-bcd9-6361d8646ddf is now active with session: 4e9c613a-cd76-4b53-833b-cac88773cf53","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.19639979Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196412613Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756,"session_id":"4e9c613a-cd76-4b53-833b-cac88773cf53"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196424355Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756} {"msg":"Activated sub_volume 432ead01-768f-4cd9-bcd9-6361d8646ddf","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:40.196435208Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 522 len 40960 data: 13 13 15 14 15 15 13 13 10 13 02/30 Read block 327 len 24576 03/30 Write block 178 len 8192 data: 12 14 04/30 Write block 167 len 12288 data: 12 12 15 05/30 Read block 39 len 32768 06/30 Write block 402 len 20480 data: 11 12 12 13 14 07/30 Read block 140 len 16384 08/30 Write block 181 len 32768 data: 14 14 14 18 16 12 11 11 09/30 Read block 493 len 32768 10/30 Write block 230 len 8192 data: 15 11 11/30 Read block 377 len 12288 12/30 Read block 167 len 36864 13/30 Read block 268 len 40960 14/30 Read block 384 len 4096 15/30 Read block 182 len 4096 16/30 Write block 326 len 8192 data: 13 13 17/30 Read block 81 len 36864 18/30 Read block 411 len 4096 19/30 Flush 20/30 Write block 23 len 32768 data: 7 5 3 6 6 9 9 7 21/30 Read block 146 len 20480 22/30 Write block 78 len 4096 data: 13 23/30 Read block 378 len 20480 24/30 Read block 76 len 28672 25/30 Read block 51 len 28672 26/30 Read block 416 len 8192 27/30 Write block 245 len 28672 data: 6 10 11 10 11 10 12 28/30 Read block 187 len 28672 29/30 Read block 392 len 4096 30/30 Read block 20 len 36864 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 8 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 10 10 11 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:40.217645434Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1756} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 228 228 228 F F F 1 020-039 124 124 124 202 202 202 T T T 2 040-059 140 140 140 233 233 233 F F F 3 060-079 141 141 141 235 235 235 T T T 4 080-099 141 141 141 235 235 235 F F F 5 100-119 138 138 138 231 231 231 F F F 6 120-139 141 141 141 235 235 235 F F F 7 140-159 139 139 139 232 232 232 F F F 8 160-179 142 142 142 236 236 236 F F F 9 180-199 142 142 142 236 236 236 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 142 142 142 236 236 236 F F F 12 240-259 140 140 140 234 234 234 T T T 13 260-279 135 135 135 223 223 223 F F F 14 280-299 140 140 140 233 233 233 F F F 15 300-319 141 141 141 235 235 235 F F F 16 320-339 142 142 142 236 236 236 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 134 134 134 220 220 220 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 142 142 142 236 236 236 F F F 21 420-439 140 140 140 234 234 234 F F F 22 440-459 140 140 140 234 234 234 F F F 23 460-479 141 141 141 235 235 235 F F F 24 480-499 140 140 140 234 234 234 F F F 25 500-519 135 135 135 223 223 223 F F F 26 520-539 142 142 142 236 236 236 F F F 27 540-559 140 140 140 234 234 234 F F F 28 560-579 141 141 141 235 235 235 F F F 29 580-599 136 136 136 227 227 227 F F F Max gen: 142, Max flush: 236 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 228 228 228 F F F 1 020-039 143 143 143 237 237 237 F F F 2 040-059 140 140 140 233 233 233 F F F 3 060-079 143 143 143 237 237 237 F F F 4 080-099 141 141 141 235 235 235 F F F 5 100-119 138 138 138 231 231 231 F F F 6 120-139 141 141 141 235 235 235 F F F 7 140-159 139 139 139 232 232 232 F F F 8 160-179 142 142 142 236 236 236 F F F 9 180-199 142 142 142 236 236 236 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 142 142 142 236 236 236 F F F 12 240-259 143 143 143 237 237 237 F F F 13 260-279 135 135 135 223 223 223 F F F 14 280-299 140 140 140 233 233 233 F F F 15 300-319 141 141 141 235 235 235 F F F 16 320-339 142 142 142 236 236 236 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 134 134 134 220 220 220 F F F 19 380-399 118 118 118 193 193 193 F F F 20 400-419 142 142 142 236 236 236 F F F 21 420-439 140 140 140 234 234 234 F F F 22 440-459 140 140 140 234 234 234 F F F 23 460-479 141 141 141 235 235 235 F F F 24 480-499 140 140 140 234 234 234 F F F 25 500-519 135 135 135 223 223 223 F F F 26 520-539 142 142 142 236 236 236 F F F 27 540-559 140 140 140 234 234 234 F F F 28 560-579 141 141 141 235 235 235 F F F 29 580-599 136 136 136 227 227 227 F F F Max gen: 143, Max flush: 237 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.461971604Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.508265853Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.509112456Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.509171678Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.509190092Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.509208306Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764} {"msg":"Crucible a1040834-f1b8-45f9-9757-1875406b8409 has session id: dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.509795293Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Upstairs opts: Upstairs UUID: a1040834-f1b8-45f9-9757-1875406b8409, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.509847484Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Crucible stats registered with UUID: a1040834-f1b8-45f9-9757-1875406b8409","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.509877561Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:54.509903186Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.510498975Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.510526221Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764} {"msg":"a1040834-f1b8-45f9-9757-1875406b8409 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.510605188Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.510632033Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"0","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.51066237Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"0","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.510678343Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"1","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.510693017Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"1","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.51071042Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"2","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.510725734Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"2","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.510748809Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"io task","client":"2","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.510775735Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"io task","client":"0","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"ds_connection connected from Ok(127.0.0.1:36049)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.51080089Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"io task","client":"2","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.510827726Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"io task","client":"1","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"ds_connection connected from Ok(127.0.0.1:52614)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.510853891Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"io task","client":"0","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57781)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.511014566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"io task","client":"1","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.511526957Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"1","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.511567125Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"0","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.512247002Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"0","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.512338512Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"1","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.51293254Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.513030231Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.513855809Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"2","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.514403147Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"2","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"[0]R flush_numbers[0..12]: [228, 237, 233, 237, 235, 231, 235, 232, 236, 236, 228, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.514938673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"[0]R generation[0..12]: [137, 143, 140, 143, 141, 138, 141, 139, 142, 142, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.514967769Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.514986573Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"[1]R flush_numbers[0..12]: [228, 237, 233, 237, 235, 231, 235, 232, 236, 236, 228, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515003727Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"[1]R generation[0..12]: [137, 143, 140, 143, 141, 138, 141, 139, 142, 142, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.51502041Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515037544Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"[2]R flush_numbers[0..12]: [228, 237, 233, 237, 235, 231, 235, 232, 236, 236, 228, 236]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515054358Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"[2]R generation[0..12]: [137, 143, 140, 143, 141, 138, 141, 139, 142, 142, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515070911Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515087545Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:28:54.515102948Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515127434Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515142997Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515163961Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"":"downstairs","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515180655Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515195178Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"0","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515211121Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"1","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515226765Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"client":"2","":"downstairs","session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515242668Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515257351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"a1040834-f1b8-45f9-9757-1875406b8409 is now active with session: dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515272815Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515287178Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764,"session_id":"dbe424cf-7e5f-46c7-89ec-a73b4c6ac36f"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515301801Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764} {"msg":"Activated sub_volume a1040834-f1b8-45f9-9757-1875406b8409","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:28:54.515315244Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 556 len 40960 data: 11 11 13 13 16 15 17 17 16 15 02/30 Write block 153 len 8192 data: 14 17 03/30 Read block 294 len 4096 04/30 Read block 374 len 12288 05/30 Read block 296 len 4096 06/30 Read block 486 len 32768 07/30 Read block 433 len 20480 08/30 Read block 216 len 16384 09/30 Read block 586 len 20480 10/30 Write block 472 len 8192 data: 11 12 11/30 Write block 388 len 28672 data: 10 10 11 6 6 10 10 12/30 Write block 182 len 24576 data: 15 15 19 17 13 12 13/30 Read block 531 len 36864 14/30 Write block 109 len 40960 data: 6 5 6 7 9 12 15 15 13 15 15/30 Read block 344 len 24576 16/30 Write block 460 len 16384 data: 10 11 10 11 17/30 Read block 435 len 12288 18/30 Read block 200 len 36864 19/30 Flush 20/30 Write block 138 len 32768 data: 11 13 12 11 9 11 11 12 21/30 Write block 424 len 16384 data: 9 7 10 12 22/30 Flush 23/30 Flush 24/30 Write block 365 len 8192 data: 9 6 25/30 Write block 284 len 28672 data: 13 15 14 13 13 12 15 26/30 Read block 224 len 32768 27/30 Read block 87 len 16384 28/30 Write block 299 len 20480 data: 10 12 12 14 14 29/30 Write block 61 len 8192 data: 10 11 30/30 Write block 228 len 28672 data: 12 11 16 12 11 14 13 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:28:54.535989825Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1764} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 228 228 228 F F F 1 020-039 143 143 143 237 237 237 F F F 2 040-059 140 140 140 233 233 233 F F F 3 060-079 143 143 143 237 237 237 T T T 4 080-099 141 141 141 235 235 235 F F F 5 100-119 144 144 144 238 238 238 F F F 6 120-139 144 144 144 239 239 239 F F F 7 140-159 144 144 144 239 239 239 F F F 8 160-179 142 142 142 236 236 236 F F F 9 180-199 144 144 144 238 238 238 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 142 142 142 236 236 236 T T T 12 240-259 143 143 143 237 237 237 F F F 13 260-279 135 135 135 223 223 223 F F F 14 280-299 140 140 140 233 233 233 T T T 15 300-319 141 141 141 235 235 235 T T T 16 320-339 142 142 142 236 236 236 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 134 134 134 220 220 220 T T T 19 380-399 144 144 144 238 238 238 F F F 20 400-419 142 142 142 236 236 236 F F F 21 420-439 144 144 144 239 239 239 F F F 22 440-459 140 140 140 234 234 234 F F F 23 460-479 144 144 144 238 238 238 F F F 24 480-499 140 140 140 234 234 234 F F F 25 500-519 135 135 135 223 223 223 F F F 26 520-539 142 142 142 236 236 236 F F F 27 540-559 144 144 144 238 238 238 F F F 28 560-579 144 144 144 238 238 238 F F F 29 580-599 136 136 136 227 227 227 F F F Max gen: 144, Max flush: 239 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 228 228 228 F F F 1 020-039 143 143 143 237 237 237 F F F 2 040-059 140 140 140 233 233 233 F F F 3 060-079 145 145 145 240 240 240 F F F 4 080-099 141 141 141 235 235 235 F F F 5 100-119 144 144 144 238 238 238 F F F 6 120-139 144 144 144 239 239 239 F F F 7 140-159 144 144 144 239 239 239 F F F 8 160-179 142 142 142 236 236 236 F F F 9 180-199 144 144 144 238 238 238 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 145 145 145 240 240 240 F F F 12 240-259 143 143 143 237 237 237 F F F 13 260-279 135 135 135 223 223 223 F F F 14 280-299 145 145 145 240 240 240 F F F 15 300-319 145 145 145 240 240 240 F F F 16 320-339 142 142 142 236 236 236 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 145 145 145 240 240 240 F F F 19 380-399 144 144 144 238 238 238 F F F 20 400-419 142 142 142 236 236 236 F F F 21 420-439 144 144 144 239 239 239 F F F 22 440-459 140 140 140 234 234 234 F F F 23 460-479 144 144 144 238 238 238 F F F 24 480-499 140 140 140 234 234 234 F F F 25 500-519 135 135 135 223 223 223 F F F 26 520-539 142 142 142 236 236 236 F F F 27 540-559 144 144 144 238 238 238 F F F 28 560-579 144 144 144 238 238 238 F F F 29 580-599 136 136 136 227 227 227 F F F Max gen: 145, Max flush: 240 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.777172845Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.822351343Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.823085001Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.823127901Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.823143234Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.823157737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772} {"msg":"Crucible 05dee2d6-3937-4e8b-814c-b8e2c7d4f4f7 has session id: 164558ad-bace-468c-b40c-46d65637da11","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.823725149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Upstairs opts: Upstairs UUID: 05dee2d6-3937-4e8b-814c-b8e2c7d4f4f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.823770489Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Crucible stats registered with UUID: 05dee2d6-3937-4e8b-814c-b8e2c7d4f4f7","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.823794775Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:08.823816789Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824221907Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824246332Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772} {"msg":"05dee2d6-3937-4e8b-814c-b8e2c7d4f4f7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824303574Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824321388Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"0","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824351455Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"0","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.82437509Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"1","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824402266Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"1","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824429752Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"2","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824453937Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"2","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824475742Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"io task","client":"2","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824501967Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"io task","client":"0","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824530263Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"io task","client":"1","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"ds_connection connected from Ok(127.0.0.1:59036)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824571442Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"io task","client":"0","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"ds_connection connected from Ok(127.0.0.1:46374)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824602829Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"io task","client":"1","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"ds_connection connected from Ok(127.0.0.1:47277)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.824783328Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"io task","client":"2","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.825289797Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"0","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.825329536Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"1","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.825887926Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"0","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.826053292Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"1","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.826475173Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.826588297Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.82669256Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"2","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827186336Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"2","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"[0]R flush_numbers[0..12]: [228, 237, 233, 240, 235, 238, 239, 239, 236, 238, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827685574Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"[0]R generation[0..12]: [137, 143, 140, 145, 141, 144, 144, 144, 142, 144, 137, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827707299Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827722292Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"[1]R flush_numbers[0..12]: [228, 237, 233, 240, 235, 238, 239, 239, 236, 238, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827736135Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"[1]R generation[0..12]: [137, 143, 140, 145, 141, 144, 144, 144, 142, 144, 137, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827750568Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827765261Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"[2]R flush_numbers[0..12]: [228, 237, 233, 240, 235, 238, 239, 239, 236, 238, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827783055Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"[2]R generation[0..12]: [137, 143, 140, 145, 141, 144, 144, 144, 142, 144, 137, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827797378Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827811781Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827824454Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827837217Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827849739Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:29:08.827862382Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"":"downstairs","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827877596Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827889268Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"0","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827902351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"1","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827915204Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"client":"2","":"downstairs","session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827929087Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827941269Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"05dee2d6-3937-4e8b-814c-b8e2c7d4f4f7 is now active with session: 164558ad-bace-468c-b40c-46d65637da11","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827953692Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827965324Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772,"session_id":"164558ad-bace-468c-b40c-46d65637da11"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.827978167Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772} {"msg":"Activated sub_volume 05dee2d6-3937-4e8b-814c-b8e2c7d4f4f7","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:08.82799133Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 265 len 12288 data: 16 14 13 02/30 Write block 44 len 36864 data: 17 13 14 14 17 17 15 12 13 03/30 Write block 18 len 40960 data: 6 6 7 6 8 8 6 4 7 7 04/30 Read block 57 len 4096 05/30 Read block 316 len 40960 06/30 Write block 526 len 8192 data: 16 16 07/30 Write block 55 len 8192 data: 13 11 08/30 Flush 09/30 Write block 420 len 32768 data: 7 5 6 8 10 8 11 13 10/30 Flush 11/30 Write block 501 len 36864 data: 15 14 14 12 11 9 10 11 13 12/30 Write block 42 len 16384 data: 11 15 18 14 13/30 Read block 72 len 20480 14/30 Read block 271 len 32768 15/30 Read block 354 len 32768 16/30 Read block 171 len 32768 17/30 Write block 559 len 4096 data: 14 18/30 Read block 401 len 4096 19/30 Write block 583 len 20480 data: 10 13 14 10 11 20/30 Read block 327 len 32768 21/30 Read block 118 len 4096 22/30 Write block 362 len 36864 data: 11 11 8 10 7 7 6 5 7 23/30 Write block 179 len 40960 data: 15 15 15 16 16 20 18 14 13 12 24/30 Read block 422 len 32768 25/30 Read block 177 len 36864 26/30 Write block 86 len 8192 data: 9 8 27/30 Read block 435 len 24576 28/30 Read block 241 len 40960 29/30 Read block 278 len 36864 30/30 Read block 425 len 8192 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 9 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:08.848866624Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1772} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 241 241 241 F F F 1 020-039 146 146 146 241 241 241 F F F 2 040-059 146 146 146 241 241 241 T T T 3 060-079 145 145 145 240 240 240 F F F 4 080-099 141 141 141 235 235 235 T T T 5 100-119 144 144 144 238 238 238 F F F 6 120-139 144 144 144 239 239 239 F F F 7 140-159 144 144 144 239 239 239 F F F 8 160-179 142 142 142 236 236 236 T T T 9 180-199 144 144 144 238 238 238 T T T 10 200-219 137 137 137 228 228 228 F F F 11 220-239 145 145 145 240 240 240 F F F 12 240-259 143 143 143 237 237 237 F F F 13 260-279 146 146 146 241 241 241 F F F 14 280-299 145 145 145 240 240 240 F F F 15 300-319 145 145 145 240 240 240 F F F 16 320-339 142 142 142 236 236 236 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 145 145 145 240 240 240 T T T 19 380-399 144 144 144 238 238 238 F F F 20 400-419 142 142 142 236 236 236 F F F 21 420-439 146 146 146 242 242 242 F F F 22 440-459 140 140 140 234 234 234 F F F 23 460-479 144 144 144 238 238 238 F F F 24 480-499 140 140 140 234 234 234 F F F 25 500-519 135 135 135 223 223 223 T T T 26 520-539 146 146 146 241 241 241 F F F 27 540-559 144 144 144 238 238 238 T T T 28 560-579 144 144 144 238 238 238 F F F 29 580-599 136 136 136 227 227 227 T T T Max gen: 146, Max flush: 242 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 241 241 241 F F F 1 020-039 146 146 146 241 241 241 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 145 145 145 240 240 240 F F F 4 080-099 147 147 147 243 243 243 F F F 5 100-119 144 144 144 238 238 238 F F F 6 120-139 144 144 144 239 239 239 F F F 7 140-159 144 144 144 239 239 239 F F F 8 160-179 147 147 147 243 243 243 F F F 9 180-199 147 147 147 243 243 243 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 145 145 145 240 240 240 F F F 12 240-259 143 143 143 237 237 237 F F F 13 260-279 146 146 146 241 241 241 F F F 14 280-299 145 145 145 240 240 240 F F F 15 300-319 145 145 145 240 240 240 F F F 16 320-339 142 142 142 236 236 236 F F F 17 340-359 135 135 135 223 223 223 F F F 18 360-379 147 147 147 243 243 243 F F F 19 380-399 144 144 144 238 238 238 F F F 20 400-419 142 142 142 236 236 236 F F F 21 420-439 146 146 146 242 242 242 F F F 22 440-459 140 140 140 234 234 234 F F F 23 460-479 144 144 144 238 238 238 F F F 24 480-499 140 140 140 234 234 234 F F F 25 500-519 147 147 147 243 243 243 F F F 26 520-539 146 146 146 241 241 241 F F F 27 540-559 147 147 147 243 243 243 F F F 28 560-579 144 144 144 238 238 238 F F F 29 580-599 147 147 147 243 243 243 F F F Max gen: 147, Max flush: 243 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.106081426Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:23.148886391Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.149256011Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.180448722Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.18118251Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.181222969Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.181241333Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.181258907Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780} {"msg":"Crucible 568dcce2-f3f1-4d7c-a89c-f0236379a821 has session id: b94d4c09-e381-47b7-921c-d3a3e722c4a8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.181862147Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Upstairs opts: Upstairs UUID: 568dcce2-f3f1-4d7c-a89c-f0236379a821, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.181912437Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Crucible stats registered with UUID: 568dcce2-f3f1-4d7c-a89c-f0236379a821","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.181941134Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:23.181967079Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182410835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182439011Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780} {"msg":"568dcce2-f3f1-4d7c-a89c-f0236379a821 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182498634Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182516858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"0","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182545364Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"0","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182562868Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"1","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182577041Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"1","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182590104Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"2","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182604737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"2","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.18261738Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"io task","client":"2","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182630963Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"io task","client":"0","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182644175Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"io task","client":"1","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:40122)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182658689Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"io task","client":"0","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:55976)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182762761Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"io task","client":"1","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:50083)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.182918925Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"io task","client":"2","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.183462242Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"1","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.183489728Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"2","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.183735291Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"0","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.184235879Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"1","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.184277668Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"2","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.184315106Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"0","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.184815694Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.184918836Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"[0]R flush_numbers[0..12]: [241, 241, 243, 240, 243, 238, 239, 239, 243, 243, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185021899Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"[0]R generation[0..12]: [146, 146, 147, 145, 147, 144, 144, 144, 147, 147, 137, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185050545Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185067228Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"[1]R flush_numbers[0..12]: [241, 241, 243, 240, 243, 238, 239, 239, 243, 243, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185082121Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"[1]R generation[0..12]: [146, 146, 147, 145, 147, 144, 144, 144, 147, 147, 137, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185095775Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185109377Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"[2]R flush_numbers[0..12]: [241, 241, 243, 240, 243, 238, 239, 239, 243, 243, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.18512325Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"[2]R generation[0..12]: [146, 146, 147, 145, 147, 144, 144, 144, 147, 147, 137, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185137413Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185151276Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185163899Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185176682Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185193545Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185207039Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"":"downstairs","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185219891Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185231364Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"0","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185244276Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"1","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:29:23.185256909Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"client":"2","":"downstairs","session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185275813Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185287526Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"568dcce2-f3f1-4d7c-a89c-f0236379a821 is now active with session: b94d4c09-e381-47b7-921c-d3a3e722c4a8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185300059Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185311551Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780,"session_id":"b94d4c09-e381-47b7-921c-d3a3e722c4a8"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185323493Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780} {"msg":"Activated sub_volume 568dcce2-f3f1-4d7c-a89c-f0236379a821","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:23.185334296Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 430 len 24576 data: 14 14 13 13 12 11 02/30 Read block 316 len 28672 03/30 Write block 60 len 36864 data: 10 11 12 10 12 11 11 12 10 04/30 Read block 201 len 4096 05/30 Read block 540 len 36864 06/30 Write block 13 len 32768 data: 7 7 6 7 6 7 7 8 07/30 Read block 544 len 36864 08/30 Write block 356 len 40960 data: 8 10 13 13 14 10 12 12 9 11 09/30 Read block 152 len 4096 10/30 Write block 260 len 4096 data: 11 11/30 Read block 229 len 40960 12/30 Read block 440 len 32768 13/30 Read block 33 len 12288 14/30 Read block 157 len 36864 15/30 Write block 567 len 20480 data: 16 15 15 12 13 16/30 Write block 377 len 40960 data: 7 9 11 10 11 10 10 8 9 10 17/30 Flush 18/30 Write block 100 len 20480 data: 7 7 7 6 7 19/30 Write block 20 len 20480 data: 9 7 9 9 7 20/30 Read block 552 len 36864 21/30 Flush 22/30 Flush 23/30 Write block 419 len 20480 data: 8 8 6 7 9 24/30 Write block 108 len 24576 data: 8 7 6 7 8 10 25/30 Read block 224 len 8192 26/30 Read block 369 len 28672 27/30 Write block 492 len 24576 data: 15 13 11 12 12 11 28/30 Read block 564 len 40960 29/30 Read block 307 len 12288 30/30 Read block 109 len 24576 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:23.207665494Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1780} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 244 244 244 F F F 1 020-039 148 148 148 245 245 245 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 148 148 148 244 244 244 F F F 4 080-099 147 147 147 243 243 243 F F F 5 100-119 148 148 148 245 245 245 T T T 6 120-139 144 144 144 239 239 239 F F F 7 140-159 144 144 144 239 239 239 F F F 8 160-179 147 147 147 243 243 243 F F F 9 180-199 147 147 147 243 243 243 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 145 145 145 240 240 240 F F F 12 240-259 143 143 143 237 237 237 F F F 13 260-279 148 148 148 244 244 244 F F F 14 280-299 145 145 145 240 240 240 F F F 15 300-319 145 145 145 240 240 240 F F F 16 320-339 142 142 142 236 236 236 F F F 17 340-359 148 148 148 244 244 244 F F F 18 360-379 148 148 148 244 244 244 F F F 19 380-399 148 148 148 244 244 244 F F F 20 400-419 142 142 142 236 236 236 T T T 21 420-439 148 148 148 244 244 244 T T T 22 440-459 140 140 140 234 234 234 F F F 23 460-479 144 144 144 238 238 238 F F F 24 480-499 140 140 140 234 234 234 T T T 25 500-519 147 147 147 243 243 243 F F F 26 520-539 146 146 146 241 241 241 F F F 27 540-559 147 147 147 243 243 243 F F F 28 560-579 148 148 148 244 244 244 F F F 29 580-599 147 147 147 243 243 243 F F F Max gen: 148, Max flush: 245 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 244 244 244 F F F 1 020-039 148 148 148 245 245 245 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 148 148 148 244 244 244 F F F 4 080-099 147 147 147 243 243 243 F F F 5 100-119 149 149 149 246 246 246 F F F 6 120-139 144 144 144 239 239 239 F F F 7 140-159 144 144 144 239 239 239 F F F 8 160-179 147 147 147 243 243 243 F F F 9 180-199 147 147 147 243 243 243 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 145 145 145 240 240 240 F F F 12 240-259 143 143 143 237 237 237 F F F 13 260-279 148 148 148 244 244 244 F F F 14 280-299 145 145 145 240 240 240 F F F 15 300-319 145 145 145 240 240 240 F F F 16 320-339 142 142 142 236 236 236 F F F 17 340-359 148 148 148 244 244 244 F F F 18 360-379 148 148 148 244 244 244 F F F 19 380-399 148 148 148 244 244 244 F F F 20 400-419 149 149 149 246 246 246 F F F 21 420-439 149 149 149 246 246 246 F F F 22 440-459 140 140 140 234 234 234 F F F 23 460-479 144 144 144 238 238 238 F F F 24 480-499 149 149 149 246 246 246 F F F 25 500-519 147 147 147 243 243 243 F F F 26 520-539 146 146 146 241 241 241 F F F 27 540-559 147 147 147 243 243 243 F F F 28 560-579 148 148 148 244 244 244 F F F 29 580-599 147 147 147 243 243 243 F F F Max gen: 149, Max flush: 246 Begin loop 75 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.472842542Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:27.515788168Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.516223072Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.547560704Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.548189579Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.548225477Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.54824036Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.548255073Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788} {"msg":"Crucible 24a84379-5970-4d0f-9e3d-d8af99974526 has session id: 17809dac-15b5-4821-8f00-dfbbbfa7f4dd","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.54897973Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Upstairs opts: Upstairs UUID: 24a84379-5970-4d0f-9e3d-d8af99974526, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549012387Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Crucible stats registered with UUID: 24a84379-5970-4d0f-9e3d-d8af99974526","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549032361Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:27.549046274Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549562405Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.54958407Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788} {"msg":"24a84379-5970-4d0f-9e3d-d8af99974526 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549650864Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549668538Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"0","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549698455Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"0","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.54972285Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"1","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549750556Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"1","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549778662Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"2","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549807478Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"2","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549837565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549866441Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549891727Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:47625)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549926074Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:48036)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.54995376Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:35125)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.549978355Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.550593048Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"1","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.550633747Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"2","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.55078612Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"0","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.551295859Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"2","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.551460305Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"1","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.551488701Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"0","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.551969715Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552089671Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"[0]R flush_numbers[0..12]: [244, 245, 243, 244, 243, 246, 239, 239, 243, 243, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552213137Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"[0]R generation[0..12]: [148, 148, 147, 148, 147, 149, 144, 144, 147, 147, 137, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552232292Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552246515Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"[1]R flush_numbers[0..12]: [244, 245, 243, 244, 243, 246, 239, 239, 243, 243, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552260167Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"[1]R generation[0..12]: [148, 148, 147, 148, 147, 149, 144, 144, 147, 147, 137, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552286063Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552312689Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"[2]R flush_numbers[0..12]: [244, 245, 243, 244, 243, 246, 239, 239, 243, 243, 228, 240]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552338755Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"[2]R generation[0..12]: [148, 148, 147, 148, 147, 149, 144, 144, 147, 147, 137, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.5523653Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552398327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552422133Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552436296Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552449288Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552471053Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"":"downstairs","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552495408Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552519014Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"0","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552534847Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"1","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.55254958Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"client":"2","":"downstairs","session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552573345Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552599701Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"24a84379-5970-4d0f-9e3d-d8af99974526 is now active with session: 17809dac-15b5-4821-8f00-dfbbbfa7f4dd","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552621716Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552634178Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788,"session_id":"17809dac-15b5-4821-8f00-dfbbbfa7f4dd"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:27.552650432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788} {"msg":"Activated sub_volume 24a84379-5970-4d0f-9e3d-d8af99974526","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-17T22:29:27.552669666Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 319 len 16384 data: 9 11 11 11 02/30 Write block 459 len 28672 data: 10 11 12 11 12 12 11 03/30 Read block 140 len 4096 04/30 Read block 397 len 20480 05/30 Write block 534 len 16384 data: 11 11 9 11 06/30 Read block 327 len 12288 07/30 Read block 406 len 12288 08/30 Write block 590 len 32768 data: 12 13 11 13 11 9 10 8 09/30 Read block 262 len 32768 10/30 Write block 351 len 12288 data: 8 7 8 11/30 Write block 19 len 24576 data: 8 10 8 10 10 8 12/30 Read block 468 len 20480 13/30 Read block 278 len 8192 14/30 Read block 327 len 16384 15/30 Read block 450 len 12288 16/30 Write block 288 len 8192 data: 14 13 17/30 Write block 10 len 40960 data: 11 9 10 8 8 7 8 7 8 9 18/30 Read block 445 len 16384 19/30 Write block 569 len 8192 data: 16 13 20/30 Flush 21/30 Write block 305 len 20480 data: 12 13 13 14 13 22/30 Write block 358 len 28672 data: 14 14 15 11 13 13 10 23/30 Write block 577 len 32768 data: 13 14 14 10 11 11 11 14 24/30 Write block 117 len 16384 data: 14 16 15 13 25/30 Write block 232 len 8192 data: 12 15 26/30 Read block 282 len 24576 27/30 Read block 451 len 4096 28/30 Write block 319 len 28672 data: 10 12 12 12 12 12 14 29/30 Write block 90 len 8192 data: 11 11 30/30 Write block 589 len 28672 data: 13 13 14 12 14 12 10 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 5 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 2 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 3 5 Done 9 9 7 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:27.571110605Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1788} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 150 150 150 247 247 247 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 148 148 148 244 244 244 F F F 4 080-099 147 147 147 243 243 243 T T T 5 100-119 149 149 149 246 246 246 T T T 6 120-139 144 144 144 239 239 239 T T T 7 140-159 144 144 144 239 239 239 F F F 8 160-179 147 147 147 243 243 243 F F F 9 180-199 147 147 147 243 243 243 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 145 145 145 240 240 240 T T T 12 240-259 143 143 143 237 237 237 F F F 13 260-279 148 148 148 244 244 244 F F F 14 280-299 150 150 150 247 247 247 F F F 15 300-319 150 150 150 247 247 247 T T T 16 320-339 150 150 150 247 247 247 T T T 17 340-359 150 150 150 247 247 247 T T T 18 360-379 148 148 148 244 244 244 T T T 19 380-399 148 148 148 244 244 244 F F F 20 400-419 149 149 149 246 246 246 F F F 21 420-439 149 149 149 246 246 246 F F F 22 440-459 150 150 150 247 247 247 F F F 23 460-479 150 150 150 247 247 247 F F F 24 480-499 149 149 149 246 246 246 F F F 25 500-519 147 147 147 243 243 243 F F F 26 520-539 150 150 150 247 247 247 F F F 27 540-559 147 147 147 243 243 243 F F F 28 560-579 150 150 150 247 247 247 T T T 29 580-599 150 150 150 247 247 247 T T T Max gen: 150, Max flush: 247 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 247 247 247 F F F 1 020-039 150 150 150 247 247 247 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 148 148 148 244 244 244 F F F 4 080-099 151 151 151 248 248 248 F F F 5 100-119 151 151 151 248 248 248 F F F 6 120-139 151 151 151 248 248 248 F F F 7 140-159 144 144 144 239 239 239 F F F 8 160-179 147 147 147 243 243 243 F F F 9 180-199 147 147 147 243 243 243 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 151 151 151 248 248 248 F F F 12 240-259 143 143 143 237 237 237 F F F 13 260-279 148 148 148 244 244 244 F F F 14 280-299 150 150 150 247 247 247 F F F 15 300-319 151 151 151 248 248 248 F F F 16 320-339 151 151 151 248 248 248 F F F 17 340-359 151 151 151 248 248 248 F F F 18 360-379 151 151 151 248 248 248 F F F 19 380-399 148 148 148 244 244 244 F F F 20 400-419 149 149 149 246 246 246 F F F 21 420-439 149 149 149 246 246 246 F F F 22 440-459 150 150 150 247 247 247 F F F 23 460-479 150 150 150 247 247 247 F F F 24 480-499 149 149 149 246 246 246 F F F 25 500-519 147 147 147 243 243 243 F F F 26 520-539 150 150 150 247 247 247 F F F 27 540-559 147 147 147 243 243 243 F F F 28 560-579 151 151 151 248 248 248 F F F 29 580-599 151 151 151 248 248 248 F F F Max gen: 151, Max flush: 248 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.865367566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.909452609Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.910238668Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.910287719Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.910305203Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.910322646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796} {"msg":"Crucible 451e29f2-29fe-4584-9655-194e2bb26bf4 has session id: 869f7bce-c874-4b26-9ddb-4f17b4b0a93b","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.91098826Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"Upstairs opts: Upstairs UUID: 451e29f2-29fe-4584-9655-194e2bb26bf4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911041762Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"Crucible stats registered with UUID: 451e29f2-29fe-4584-9655-194e2bb26bf4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911066047Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:31.911082281Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911486268Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911519475Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796} {"msg":"451e29f2-29fe-4584-9655-194e2bb26bf4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911571266Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911597492Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"0","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.91163512Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"0","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911662606Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"1","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911697033Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"1","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.91172712Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"2","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911756116Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"2","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911783362Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911810888Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911837113Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37743)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.911864599Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:31.911892485Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43180)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.91191503Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:31.911937505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:31.911964381Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.912047499Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.912503937Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"1","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.912541865Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"0","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.913171441Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"1","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.913198957Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"0","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.91376735Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:31.913803037Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.913017659Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"ds_connection connected from Ok(127.0.0.1:65454)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.913082693Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.914243543Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"2","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.914754534Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"2","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"[0]R flush_numbers[0..12]: [247, 247, 243, 244, 248, 248, 248, 239, 243, 243, 228, 248]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915332688Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"[0]R generation[0..12]: [150, 150, 147, 148, 151, 151, 151, 144, 147, 147, 137, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915358384Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915377078Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"[1]R flush_numbers[0..12]: [247, 247, 243, 244, 248, 248, 248, 239, 243, 243, 228, 248]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915394072Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"[1]R generation[0..12]: [150, 150, 147, 148, 151, 151, 151, 144, 147, 147, 137, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915410735Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915427909Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"[2]R flush_numbers[0..12]: [247, 247, 243, 244, 248, 248, 248, 239, 243, 243, 228, 248]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915448973Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"[2]R generation[0..12]: [150, 150, 147, 148, 151, 151, 151, 144, 147, 147, 137, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915466207Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915483011Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915498474Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"Wait for a query_work_queue command to finish before sending IO msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915521319Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915536962Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915553176Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"":"downstairs","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915569059Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915583512Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"0","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915599596Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"1","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915615899Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"client":"2","":"downstairs","session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915631963Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915646616Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"451e29f2-29fe-4584-9655-194e2bb26bf4 is now active with session: 869f7bce-c874-4b26-9ddb-4f17b4b0a93b","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915661969Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915676312Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796,"session_id":"869f7bce-c874-4b26-9ddb-4f17b4b0a93b"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915693296Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796} {"msg":"Activated sub_volume 451e29f2-29fe-4584-9655-194e2bb26bf4","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:41.915707289Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796}Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 153 len 8192 data: 15 18 02/30 Read block 517 len 4096 03/30 Read block 586 len 24576 04/30 Write block 568 len 24576 data: 16 17 14 14 11 12 05/30 Read block 251 len 28672 06/30 Flush 07/30 Write block 379 len 32768 data: 12 11 12 11 11 9 10 11 08/30 Read block 557 len 36864 09/30 Read block 201 len 8192 10/30 Read block 372 len 8192 11/30 Write block 412 len 36864 data: 10 10 9 6 7 7 8 9 9 12/30 Read block 492 len 32768 13/30 Write block 555 len 16384 data: 11 12 12 14 14/30 Write block 484 len 4096 data: 10 15/30 Read block 584 len 24576 16/30 Write block 548 len 16384 data: 13 13 12 11 17/30 Read block 142 len 20480 18/30 Read block 130 len 4096 19/30 Write block 99 len 40960 data: 10 8 8 8 7 8 7 7 7 9 20/30 Read block 291 len 40960 21/30 Write block 0 len 40960 data: 5 7 9 10 11 11 13 14 12 10 22/30 Flush 23/30 Write block 172 len 32768 data: 12 10 9 8 11 11 13 16 24/30 Read block 362 len 24576 25/30 Read block 397 len 36864 26/30 Read block 74 len 20480 27/30 Write block 596 len 8192 data: 11 9 28/30 Read block 222 len 24576 29/30 Write block 62 len 8192 data: 13 11 30/30 Write block 460 len 24576 data: 12 13 12 13 13 12 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:41.936671652Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1796} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 250 250 250 F F F 1 020-039 150 150 150 247 247 247 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 148 148 148 244 244 244 T T T 4 080-099 152 152 152 250 250 250 F F F 5 100-119 152 152 152 250 250 250 F F F 6 120-139 151 151 151 248 248 248 F F F 7 140-159 152 152 152 249 249 249 F F F 8 160-179 147 147 147 243 243 243 T T T 9 180-199 147 147 147 243 243 243 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 151 151 151 248 248 248 F F F 12 240-259 143 143 143 237 237 237 F F F 13 260-279 148 148 148 244 244 244 F F F 14 280-299 150 150 150 247 247 247 F F F 15 300-319 151 151 151 248 248 248 F F F 16 320-339 151 151 151 248 248 248 F F F 17 340-359 151 151 151 248 248 248 F F F 18 360-379 152 152 152 250 250 250 F F F 19 380-399 152 152 152 250 250 250 F F F 20 400-419 152 152 152 250 250 250 F F F 21 420-439 152 152 152 250 250 250 F F F 22 440-459 150 150 150 247 247 247 F F F 23 460-479 150 150 150 247 247 247 T T T 24 480-499 152 152 152 250 250 250 F F F 25 500-519 147 147 147 243 243 243 F F F 26 520-539 150 150 150 247 247 247 F F F 27 540-559 152 152 152 250 250 250 F F F 28 560-579 152 152 152 249 249 249 F F F 29 580-599 151 151 151 248 248 248 T T T Max gen: 152, Max flush: 250 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 250 250 250 F F F 1 020-039 150 150 150 247 247 247 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 153 153 153 251 251 251 F F F 4 080-099 152 152 152 250 250 250 F F F 5 100-119 152 152 152 250 250 250 F F F 6 120-139 151 151 151 248 248 248 F F F 7 140-159 152 152 152 249 249 249 F F F 8 160-179 153 153 153 251 251 251 F F F 9 180-199 147 147 147 243 243 243 F F F 10 200-219 137 137 137 228 228 228 F F F 11 220-239 151 151 151 248 248 248 F F F 12 240-259 143 143 143 237 237 237 F F F 13 260-279 148 148 148 244 244 244 F F F 14 280-299 150 150 150 247 247 247 F F F 15 300-319 151 151 151 248 248 248 F F F 16 320-339 151 151 151 248 248 248 F F F 17 340-359 151 151 151 248 248 248 F F F 18 360-379 152 152 152 250 250 250 F F F 19 380-399 152 152 152 250 250 250 F F F 20 400-419 152 152 152 250 250 250 F F F 21 420-439 152 152 152 250 250 250 F F F 22 440-459 150 150 150 247 247 247 F F F 23 460-479 153 153 153 251 251 251 F F F 24 480-499 152 152 152 250 250 250 F F F 25 500-519 147 147 147 243 243 243 F F F 26 520-539 150 150 150 247 247 247 F F F 27 540-559 152 152 152 250 250 250 F F F 28 560-579 152 152 152 249 249 249 F F F 29 580-599 153 153 153 251 251 251 F F F Max gen: 153, Max flush: 251 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.17681549Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.220589295Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.221269852Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.221310551Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.221328965Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.221344988Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804} {"msg":"Crucible 4a2f9e64-23fe-40ae-a155-59052a0a5607 has session id: 60a2c287-dc8a-472f-bcfd-587f66b6251f","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.221943737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Upstairs opts: Upstairs UUID: 4a2f9e64-23fe-40ae-a155-59052a0a5607, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.222015973Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Crucible stats registered with UUID: 4a2f9e64-23fe-40ae-a155-59052a0a5607","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.22204863Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:56.222076276Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.222551889Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.222578604Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804} {"msg":"4a2f9e64-23fe-40ae-a155-59052a0a5607 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.22265135Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.222669074Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"0","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.22269751Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"0","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.222711413Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"1","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.222725776Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"1","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.222739239Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"2","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.222752982Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"2","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.222773106Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.222797362Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.222821176Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39837)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.222965978Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55689)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.223074921Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:56.22311281Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:56.223139595Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:29:56.223212481Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.223387629Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.223835215Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"2","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.223863621Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"0","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.224543728Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"0","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.224578836Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"2","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.225166733Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:29:56.225197499Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.224602762Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56811)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.224951267Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.225914915Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"1","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.22649285Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"1","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"[0]R flush_numbers[0..12]: [250, 247, 243, 251, 250, 250, 248, 249, 251, 243, 228, 248]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227139369Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"[0]R generation[0..12]: [152, 150, 147, 153, 152, 152, 151, 152, 153, 147, 137, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227167955Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227184279Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"[1]R flush_numbers[0..12]: [250, 247, 243, 251, 250, 250, 248, 249, 251, 243, 228, 248]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227198432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"[1]R generation[0..12]: [152, 150, 147, 153, 152, 152, 151, 152, 153, 147, 137, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227212605Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227227048Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"[2]R flush_numbers[0..12]: [250, 247, 243, 251, 250, 250, 248, 249, 251, 243, 228, 248]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227240681Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"[2]R generation[0..12]: [152, 150, 147, 153, 152, 152, 151, 152, 153, 147, 137, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227254434Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227272748Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227286111Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227298894Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227311416Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227323919Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"":"downstairs","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227336542Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227348264Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"0","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227361107Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"1","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.22737379Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"client":"2","":"downstairs","session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227386573Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227398375Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"4a2f9e64-23fe-40ae-a155-59052a0a5607 is now active with session: 60a2c287-dc8a-472f-bcfd-587f66b6251fWait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.22741967Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227431572Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804,"session_id":"60a2c287-dc8a-472f-bcfd-587f66b6251f"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227443475Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804} {"msg":"Activated sub_volume 4a2f9e64-23fe-40ae-a155-59052a0a5607","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:06.227454337Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 586 len 32768 data: 11 12 11 14 14 15 13 15 02/30 Read block 17 len 4096 03/30 Flush 04/30 Write block 529 len 40960 data: 14 11 14 12 12 12 12 10 12 11 05/30 Read block 127 len 16384 06/30 Read block 506 len 24576 07/30 Write block 435 len 4096 data: 12 08/30 Read block 294 len 40960 09/30 Read block 316 len 28672 10/30 Write block 501 len 28672 data: 16 15 15 13 12 10 11 11/30 Write block 23 len 36864 data: 11 9 5 8 8 10 10 8 8 12/30 Write block 66 len 16384 data: 12 13 11 12 13/30 Read block 330 len 8192 14/30 Write block 313 len 28672 data: 11 10 10 12 12 8 11 15/30 Write block 146 len 36864 data: 11 11 12 14 12 16 15 16 19 16/30 Write block 211 len 12288 data: 14 12 8 17/30 Read block 29 len 28672 18/30 Read block 586 len 20480 19/30 Read block 136 len 12288 20/30 Write block 129 len 40960 data: 10 9 10 11 11 10 10 10 8 12 21/30 Read block 539 len 16384 22/30 Write block 245 len 24576 data: 7 11 12 11 12 11 23/30 Write block 391 len 36864 data: 7 7 11 11 10 11 11 11 12 24/30 Write block 356 len 20480 data: 9 11 15 15 16 25/30 Write block 436 len 28672 data: 10 8 8 11 14 11 14 26/30 Read block 211 len 4096 27/30 Read block 202 len 40960 28/30 Read block 154 len 24576 29/30 Read block 175 len 40960 30/30 Write block 385 len 40960 data: 11 12 10 11 11 12 8 8 12 12 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 10 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:06.24920331Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1804} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 250 250 250 F F F 1 020-039 150 150 150 247 247 247 T T T 2 040-059 147 147 147 243 243 243 F F F 3 060-079 153 153 153 251 251 251 T T T 4 080-099 152 152 152 250 250 250 F F F 5 100-119 152 152 152 250 250 250 F F F 6 120-139 151 151 151 248 248 248 T T T 7 140-159 152 152 152 249 249 249 T T T 8 160-179 153 153 153 251 251 251 F F F 9 180-199 147 147 147 243 243 243 F F F 10 200-219 137 137 137 228 228 228 T T T 11 220-239 151 151 151 248 248 248 F F F 12 240-259 143 143 143 237 237 237 T T T 13 260-279 148 148 148 244 244 244 F F F 14 280-299 150 150 150 247 247 247 F F F 15 300-319 151 151 151 248 248 248 T T T 16 320-339 151 151 151 248 248 248 F F F 17 340-359 151 151 151 248 248 248 T T T 18 360-379 152 152 152 250 250 250 T T T 19 380-399 152 152 152 250 250 250 T T T 20 400-419 152 152 152 250 250 250 F F F 21 420-439 152 152 152 250 250 250 T T T 22 440-459 150 150 150 247 247 247 T T T 23 460-479 153 153 153 251 251 251 F F F 24 480-499 152 152 152 250 250 250 F F F 25 500-519 147 147 147 243 243 243 T T T 26 520-539 150 150 150 247 247 247 T T T 27 540-559 152 152 152 250 250 250 F F F 28 560-579 152 152 152 249 249 249 F F F 29 580-599 154 154 154 252 252 252 F F F Max gen: 154, Max flush: 252 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 250 250 250 F F F 1 020-039 155 155 155 253 253 253 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 155 155 155 253 253 253 F F F 4 080-099 152 152 152 250 250 250 F F F 5 100-119 152 152 152 250 250 250 F F F 6 120-139 155 155 155 253 253 253 F F F 7 140-159 155 155 155 253 253 253 F F F 8 160-179 153 153 153 251 251 251 F F F 9 180-199 147 147 147 243 243 243 F F F 10 200-219 155 155 155 253 253 253 F F F 11 220-239 151 151 151 248 248 248 F F F 12 240-259 155 155 155 253 253 253 F F F 13 260-279 148 148 148 244 244 244 F F F 14 280-299 150 150 150 247 247 247 F F F 15 300-319 155 155 155 253 253 253 F F F 16 320-339 151 151 151 248 248 248 F F F 17 340-359 155 155 155 253 253 253 F F F 18 360-379 155 155 155 253 253 253 F F F 19 380-399 155 155 155 253 253 253 F F F 20 400-419 152 152 152 250 250 250 F F F 21 420-439 155 155 155 253 253 253 F F F 22 440-459 155 155 155 253 253 253 F F F 23 460-479 153 153 153 251 251 251 F F F 24 480-499 152 152 152 250 250 250 F F F 25 500-519 155 155 155 253 253 253 F F F 26 520-539 155 155 155 253 253 253 F F F 27 540-559 152 152 152 250 250 250 F F F 28 560-579 152 152 152 249 249 249 F F F 29 580-599 154 154 154 252 252 252 F F F Max gen: 155, Max flush: 253 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.537743209Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.58168134Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.582387572Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.582430871Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.582449665Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.582464589Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812} {"msg":"Crucible 95d5f08f-b0a8-406a-9fe1-00fbe940e33a has session id: 3f4d4f55-3648-4ba0-a418-b8cc6bd1a896","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583019618Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Upstairs opts: Upstairs UUID: 95d5f08f-b0a8-406a-9fe1-00fbe940e33a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583065558Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Crucible stats registered with UUID: 95d5f08f-b0a8-406a-9fe1-00fbe940e33a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583087943Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:20.583107387Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583560535Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583587801Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812} {"msg":"95d5f08f-b0a8-406a-9fe1-00fbe940e33a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.58362938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583646373Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"0","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583673739Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"0","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583687352Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"1","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583701475Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"1","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.58372517Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"2","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583752266Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"2","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583776752Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583802767Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583828973Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"ds_connection connected from Ok(127.0.0.1:44817)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583855399Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:20.583883845Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:20.583912031Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"ds_connection connected from Ok(127.0.0.1:54245)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.583937536Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:20.583967103Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.584103532Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.584537916Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"0","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.584568102Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"1","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.585262332Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"0","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.585295439Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"1","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.585867563Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:20.585952961Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.584755125Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"ds_connection connected from Ok(127.0.0.1:49955)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.584885543Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.586096924Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"2","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.5866331Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"2","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"[0]R flush_numbers[0..12]: [250, 253, 243, 253, 250, 250, 253, 253, 251, 243, 253, 248]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.58723697Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"[0]R generation[0..12]: [152, 155, 147, 155, 152, 152, 155, 155, 153, 147, 155, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587261115Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587277089Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"[1]R flush_numbers[0..12]: [250, 253, 243, 253, 250, 250, 253, 253, 251, 243, 253, 248]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587291622Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"[1]R generation[0..12]: [152, 155, 147, 155, 152, 152, 155, 155, 153, 147, 155, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587305525Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587319458Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"[2]R flush_numbers[0..12]: [250, 253, 243, 253, 250, 250, 253, 253, 251, 243, 253, 248]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587333151Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"[2]R generation[0..12]: [152, 155, 147, 155, 152, 152, 155, 155, 153, 147, 155, 151]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587351415Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587365828Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.58737878Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587391923Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587404536Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"":"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587417179Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,""Wait for a query_work_queue command to finish before sending IO :"downstairs","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587438344Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587450056Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"0","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587463429Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"1","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587476552Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"client":"2","":"downstairs","session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587489495Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587501437Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"95d5f08f-b0a8-406a-9fe1-00fbe940e33a is now active with session: 3f4d4f55-3648-4ba0-a418-b8cc6bd1a896","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.58751397Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587525672Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812,"session_id":"3f4d4f55-3648-4ba0-a418-b8cc6bd1a896"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587537605Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812} {"msg":"Activated sub_volume 95d5f08f-b0a8-406a-9fe1-00fbe940e33a","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:30.587548557Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 474 len 32768 data: 12 13 9 9 8 8 10 9 02/30 Write block 390 len 16384 data: 13 9 9 13 03/30 Read block 552 len 4096 04/30 Write block 410 len 12288 data: 13 13 11 05/30 Read block 4 len 12288 06/30 Read block 113 len 16384 07/30 Write block 412 len 20480 data: 12 11 10 7 8 08/30 Write block 10 len 16384 data: 12 10 11 9 09/30 Write block 110 len 24576 data: 7 8 9 11 13 16 10/30 Read block 337 len 40960 11/30 Flush 12/30 Write block 331 len 16384 data: 13 14 14 16 13/30 Read block 592 len 28672 14/30 Read block 346 len 28672 15/30 Write block 313 len 36864 data: 12 11 11 13 13 9 12 13 13 16/30 Read block 332 len 8192 17/30 Write block 227 len 40960 data: 10 13 12 17 13 13 16 14 15 12 18/30 Read block 404 len 24576 19/30 Write block 131 len 24576 data: 11 12 12 11 11 11 20/30 Write block 189 len 40960 data: 10 8 7 7 8 12 11 10 12 12 21/30 Flush 22/30 Flush 23/30 Write block 160 len 28672 data: 13 12 9 13 16 19 17 24/30 Write block 196 len 36864 data: 11 13 13 11 15 17 14 12 11 25/30 Write block 64 len 8192 data: 13 12 26/30 Read block 487 len 36864 27/30 Write block 129 len 12288 data: 11 10 12 28/30 Write block 92 len 8192 data: 10 9 29/30 Read block 347 len 32768 30/30 Read block 266 len 4096 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:30.608122467Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1812} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 155 155 155 253 253 253 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 155 155 155 253 253 253 T T T 4 080-099 152 152 152 250 250 250 T T T 5 100-119 156 156 156 254 254 254 F F F 6 120-139 156 156 156 255 255 255 T T T 7 140-159 155 155 155 253 253 253 F F F 8 160-179 153 153 153 251 251 251 T T T 9 180-199 156 156 156 255 255 255 T T T 10 200-219 155 155 155 253 253 253 T T T 11 220-239 156 156 156 255 255 255 F F F 12 240-259 155 155 155 253 253 253 F F F 13 260-279 148 148 148 244 244 244 F F F 14 280-299 150 150 150 247 247 247 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 156 156 156 255 255 255 F F F 17 340-359 155 155 155 253 253 253 F F F 18 360-379 155 155 155 253 253 253 F F F 19 380-399 156 156 156 254 254 254 F F F 20 400-419 156 156 156 254 254 254 F F F 21 420-439 155 155 155 253 253 253 F F F 22 440-459 155 155 155 253 253 253 F F F 23 460-479 156 156 156 254 254 254 F F F 24 480-499 156 156 156 254 254 254 F F F 25 500-519 155 155 155 253 253 253 F F F 26 520-539 155 155 155 253 253 253 F F F 27 540-559 152 152 152 250 250 250 F F F 28 560-579 152 152 152 249 249 249 F F F 29 580-599 154 154 154 252 252 252 F F F Max gen: 156, Max flush: 255 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 155 155 155 253 253 253 F F F 2 040-059 147 147 147 243 243 243 F F F 3 060-079 157 157 157 256 256 256 F F F 4 080-099 157 157 157 256 256 256 F F F 5 100-119 156 156 156 254 254 254 F F F 6 120-139 157 157 157 256 256 256 F F F 7 140-159 155 155 155 253 253 253 F F F 8 160-179 157 157 157 256 256 256 F F F 9 180-199 157 157 157 256 256 256 F F F 10 200-219 157 157 157 256 256 256 F F F 11 220-239 156 156 156 255 255 255 F F F 12 240-259 155 155 155 253 253 253 F F F 13 260-279 148 148 148 244 244 244 F F F 14 280-299 150 150 150 247 247 247 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 156 156 156 255 255 255 F F F 17 340-359 155 155 155 253 253 253 F F F 18 360-379 155 155 155 253 253 253 F F F 19 380-399 156 156 156 254 254 254 F F F 20 400-419 156 156 156 254 254 254 F F F 21 420-439 155 155 155 253 253 253 F F F 22 440-459 155 155 155 253 253 253 F F F 23 460-479 156 156 156 254 254 254 F F F 24 480-499 156 156 156 254 254 254 F F F 25 500-519 155 155 155 253 253 253 F F F 26 520-539 155 155 155 253 253 253 F F F 27 540-559 152 152 152 250 250 250 F F F 28 560-579 152 152 152 249 249 249 F F F 29 580-599 154 154 154 252 252 252 F F F Max gen: 157, Max flush: 256 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.854595047Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.898953848Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.899690017Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.899740058Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.899757692Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.899775486Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820} {"msg":"Crucible d63eae45-ccbb-4dda-9cbe-b9fcfa6859db has session id: 0b948091-1c5a-40e9-839e-d0fd57508d78","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.900413513Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Upstairs opts: Upstairs UUID: d63eae45-ccbb-4dda-9cbe-b9fcfa6859db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.900459853Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Crucible stats registered with UUID: d63eae45-ccbb-4dda-9cbe-b9fcfa6859db","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.900484029Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:34.900504883Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901019334Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901042889Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820} {"msg":"d63eae45-ccbb-4dda-9cbe-b9fcfa6859db active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901130588Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901148302Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"0","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901176418Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"0","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901190501Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"1","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901204354Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"1","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901223458Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"2","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901248474Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"2","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901272849Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901305796Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901333762Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"ds_connection connected from Ok(127.0.0.1:44843)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.90141913Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"ds_connection connected from Ok(127.0.0.1:39764)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901443265Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"ds_connection connected from Ok(127.0.0.1:60771)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.901656222Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.902149878Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"2","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.902179984Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"0","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.902863432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"0","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.902895999Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"2","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.903464232Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.903590689Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.903764156Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"1","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.90429171Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"1","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"[0]R flush_numbers[0..12]: [254, 253, 243, 256, 256, 254, 256, 253, 256, 256, 256, 255]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.90480205Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"[0]R generation[0..12]: [156, 155, 147, 157, 157, 156, 157, 155, 157, 157, 157, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.904821104Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.904839648Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"[1]R flush_numbers[0..12]: [254, 253, 243, 256, 256, 254, 256, 253, 256, 256, 256, 255]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.904854181Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"[1]R generation[0..12]: [156, 155, 147, 157, 157, 156, 157, 155, 157, 157, 157, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.904868735Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.904882267Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"[2]R flush_numbers[0..12]: [254, 253, 243, 256, 256, 254, 256, 253, 256, 256, 256, 255]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.90489567Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"[2]R generation[0..12]: [156, 155, 147, 157, 157, 156, 157, 155, 157, 157, 157, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.904908663Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.904921936Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.904935399Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.904948262Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.904960675Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:30:34.904973367Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"":"downstairs","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.905044693Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.905059356Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"0","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.905072859Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"1","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.905088622Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"client":"2","":"downstairs","session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.905102505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.905115088Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"d63eae45-ccbb-4dda-9cbe-b9fcfa6859db is now active with session: 0b948091-1c5a-40e9-839e-d0fd57508d78","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.90512786Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.905139553Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820,"session_id":"0b948091-1c5a-40e9-839e-d0fd57508d78"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.905151385Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820} {"msg":"Activated sub_volume d63eae45-ccbb-4dda-9cbe-b9fcfa6859db","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:34.905162148Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 461 len 40960 data: 14 13 14 14 13 8 14 15 13 11 02/30 Read block 480 len 28672 03/30 Write block 206 len 28672 data: 13 11 12 15 14 15 13 04/30 Write block 69 len 32768 data: 13 14 14 15 14 19 17 15 05/30 Write block 148 len 28672 data: 13 15 13 17 16 17 20 06/30 Read block 271 len 28672 07/30 Read block 566 len 8192 08/30 Write block 534 len 40960 data: 13 13 11 13 12 13 13 12 14 16 09/30 Read block 472 len 28672 10/30 Read block 407 len 4096 11/30 Write block 384 len 28672 data: 10 12 13 11 12 12 14 12/30 Read block 309 len 28672 13/30 Write block 560 len 36864 data: 17 16 18 18 17 16 14 17 17 14/30 Read block 347 len 28672 15/30 Read block 434 len 36864 16/30 Flush 17/30 Write block 252 len 36864 data: 12 11 12 11 10 11 12 11 12 18/30 Read block 593 len 12288 19/30 Read block 32 len 24576 20/30 Read block 216 len 8192 21/30 Read block 188 len 24576 22/30 Read block 449 len 24576 23/30 Flush 24/30 Write block 292 len 12288 data: 13 10 10 25/30 Write block 59 len 8192 data: 11 11 26/30 Read block 190 len 24576 27/30 Read block 580 len 32768 28/30 Write block 363 len 12288 data: 14 11 12 29/30 Write block 395 len 4096 data: 11 30/30 Read block 260 len 16384 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 6 5 7 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:34.926615217Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1820} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 155 155 155 253 253 253 F F F 2 040-059 147 147 147 243 243 243 T T T 3 060-079 158 158 158 257 257 257 T T T 4 080-099 157 157 157 256 256 256 F F F 5 100-119 156 156 156 254 254 254 F F F 6 120-139 157 157 157 256 256 256 F F F 7 140-159 158 158 158 257 257 257 F F F 8 160-179 157 157 157 256 256 256 F F F 9 180-199 157 157 157 256 256 256 F F F 10 200-219 158 158 158 257 257 257 F F F 11 220-239 156 156 156 255 255 255 F F F 12 240-259 158 158 158 258 258 258 F F F 13 260-279 158 158 158 258 258 258 F F F 14 280-299 150 150 150 247 247 247 T T T 15 300-319 156 156 156 255 255 255 F F F 16 320-339 156 156 156 255 255 255 F F F 17 340-359 155 155 155 253 253 253 F F F 18 360-379 155 155 155 253 253 253 T T T 19 380-399 158 158 158 257 257 257 T T T 20 400-419 156 156 156 254 254 254 F F F 21 420-439 155 155 155 253 253 253 F F F 22 440-459 155 155 155 253 253 253 F F F 23 460-479 158 158 158 257 257 257 F F F 24 480-499 156 156 156 254 254 254 F F F 25 500-519 155 155 155 253 253 253 F F F 26 520-539 158 158 158 257 257 257 F F F 27 540-559 158 158 158 257 257 257 F F F 28 560-579 158 158 158 257 257 257 F F F 29 580-599 154 154 154 252 252 252 F F F Max gen: 158, Max flush: 258 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 155 155 155 253 253 253 F F F 2 040-059 159 159 159 259 259 259 F F F 3 060-079 159 159 159 259 259 259 F F F 4 080-099 157 157 157 256 256 256 F F F 5 100-119 156 156 156 254 254 254 F F F 6 120-139 157 157 157 256 256 256 F F F 7 140-159 158 158 158 257 257 257 F F F 8 160-179 157 157 157 256 256 256 F F F 9 180-199 157 157 157 256 256 256 F F F 10 200-219 158 158 158 257 257 257 F F F 11 220-239 156 156 156 255 255 255 F F F 12 240-259 158 158 158 258 258 258 F F F 13 260-279 158 158 158 258 258 258 F F F 14 280-299 159 159 159 259 259 259 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 156 156 156 255 255 255 F F F 17 340-359 155 155 155 253 253 253 F F F 18 360-379 159 159 159 259 259 259 F F F 19 380-399 159 159 159 259 259 259 F F F 20 400-419 156 156 156 254 254 254 F F F 21 420-439 155 155 155 253 253 253 F F F 22 440-459 155 155 155 253 253 253 F F F 23 460-479 158 158 158 257 257 257 F F F 24 480-499 156 156 156 254 254 254 F F F 25 500-519 155 155 155 253 253 253 F F F 26 520-539 158 158 158 257 257 257 F F F 27 540-559 158 158 158 257 257 257 F F F 28 560-579 158 158 158 257 257 257 F F F 29 580-599 154 154 154 252 252 252 F F F Max gen: 159, Max flush: 259 Begin loop 80 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.168984854Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.213524764Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.214205541Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.214249871Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.214264774Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.214279807Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828} {"msg":"Crucible cce7e24a-6cdc-4732-a52d-09966f888d61 has session id: 10bdfd51-1779-4e97-a7d9-3c8ed9890659","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.214875376Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Upstairs opts: Upstairs UUID: cce7e24a-6cdc-4732-a52d-09966f888d61, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.214919335Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Crucible stats registered with UUID: cce7e24a-6cdc-4732-a52d-09966f888d61","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.214944851Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:39.214965795Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.21535832Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215382995Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828} {"msg":"cce7e24a-6cdc-4732-a52d-09966f888d61 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215441138Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215459202Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"0","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215488048Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"0","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215506922Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"1","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215521655Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"1","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215534588Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"2","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215548731Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"2","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215561724Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215575027Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.21558843Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"ds_connection connected from Ok(127.0.0.1:58764)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215693032Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"ds_connection connected from Ok(127.0.0.1:54247)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215735641Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"ds_connection connected from Ok(127.0.0.1:34010)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.215766178Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.216418929Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"0","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.216621813Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"1","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.21702696Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"0","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.217153507Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"1","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.217646484Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.217758608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.218019014Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"2","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.218516832Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"2","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"[0]R flush_numbers[0..12]: [254, 253, 259, 259, 256, 254, 256, 257, 256, 256, 257, 255]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219038594Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"[0]R generation[0..12]: [156, 155, 159, 159, 157, 156, 157, 158, 157, 157, 158, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219062429Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219077433Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"[1]R flush_numbers[0..12]: [254, 253, 259, 259, 256, 254, 256, 257, 256, 256, 257, 255]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219091756Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"[1]R generation[0..12]: [156, 155, 159, 159, 157, 156, 157, 158, 157, 157, 158, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219105199Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219118611Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"[2]R flush_numbers[0..12]: [254, 253, 259, 259, 256, 254, 256, 257, 256, 256, 257, 255]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219131874Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"[2]R generation[0..12]: [156, 155, 159, 159, 157, 156, 157, 158, 157, 157, 158, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219145017Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.2191584Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219170943Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219188526Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Next flush: 260","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2025-03-17T22:30:39.219201819Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219222854Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"":"downstairs","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219236117Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219248299Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"0","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219261572Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"1","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219274845Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"client":"2","":"downstairs","session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219287918Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219299781Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"cce7e24a-6cdc-4732-a52d-09966f888d61 is now active with session: 10bdfd51-1779-4e97-a7d9-3c8ed9890659","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219312403Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219323886Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828,"session_id":"10bdfd51-1779-4e97-a7d9-3c8ed9890659"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219335778Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828} {"msg":"Activated sub_volume cce7e24a-6cdc-4732-a52d-09966f888d61","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:39.219347501Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 471 len 4096 data: 10 02/30 Read block 546 len 32768 03/30 Read block 51 len 16384 04/30 Read block 488 len 36864 05/30 Read block 42 len 36864 06/30 Read block 425 len 32768 07/30 Read block 439 len 24576 08/30 Read block 512 len 36864 09/30 Write block 147 len 24576 data: 12 14 16 14 18 17 10/30 Read block 559 len 8192 11/30 Read block 76 len 12288 12/30 Read block 81 len 36864 13/30 Read block 182 len 8192 14/30 Read block 145 len 16384 15/30 Read block 259 len 16384 16/30 Write block 403 len 28672 data: 13 13 14 15 17 16 13 17/30 Read block 269 len 8192 18/30 Write block 81 len 16384 data: 6 5 7 7 19/30 Write block 28 len 28672 data: 11 11 9 9 7 8 9 20/30 Read block 188 len 28672 21/30 Read block 21 len 40960 22/30 Write block 42 len 16384 data: 12 16 19 15 23/30 Read block 83 len 8192 24/30 Write block 76 len 36864 data: 16 14 14 8 9 7 6 8 8 25/30 Read block 193 len 40960 26/30 Read block 306 len 28672 27/30 Read block 301 len 12288 28/30 Read block 366 len 4096 29/30 Write block 508 len 8192 data: 12 14 30/30 Read block 195 len 12288 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 30 30 28 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:39.240835077Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1828} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 155 155 155 253 253 253 T T T 2 040-059 159 159 159 259 259 259 T T T 3 060-079 159 159 159 259 259 259 T T T 4 080-099 157 157 157 256 256 256 T T T 5 100-119 156 156 156 254 254 254 F F F 6 120-139 157 157 157 256 256 256 F F F 7 140-159 158 158 158 257 257 257 T T T 8 160-179 157 157 157 256 256 256 F F F 9 180-199 157 157 157 256 256 256 F F F 10 200-219 158 158 158 257 257 257 F F F 11 220-239 156 156 156 255 255 255 F F F 12 240-259 158 158 158 258 258 258 F F F 13 260-279 158 158 158 258 258 258 F F F 14 280-299 159 159 159 259 259 259 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 156 156 156 255 255 255 F F F 17 340-359 155 155 155 253 253 253 F F F 18 360-379 159 159 159 259 259 259 F F F 19 380-399 159 159 159 259 259 259 F F F 20 400-419 156 156 156 254 254 254 T T T 21 420-439 155 155 155 253 253 253 F F F 22 440-459 155 155 155 253 253 253 F F F 23 460-479 158 158 158 257 257 257 T T T 24 480-499 156 156 156 254 254 254 F F F 25 500-519 155 155 155 253 253 253 T T T 26 520-539 158 158 158 257 257 257 F F F 27 540-559 158 158 158 257 257 257 F F F 28 560-579 158 158 158 257 257 257 F F F 29 580-599 154 154 154 252 252 252 F F F Max gen: 159, Max flush: 259 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 160 160 160 260 260 260 F F F 2 040-059 160 160 160 260 260 260 F F F 3 060-079 160 160 160 260 260 260 F F F 4 080-099 160 160 160 260 260 260 F F F 5 100-119 156 156 156 254 254 254 F F F 6 120-139 157 157 157 256 256 256 F F F 7 140-159 160 160 160 260 260 260 F F F 8 160-179 157 157 157 256 256 256 F F F 9 180-199 157 157 157 256 256 256 F F F 10 200-219 158 158 158 257 257 257 F F F 11 220-239 156 156 156 255 255 255 F F F 12 240-259 158 158 158 258 258 258 F F F 13 260-279 158 158 158 258 258 258 F F F 14 280-299 159 159 159 259 259 259 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 156 156 156 255 255 255 F F F 17 340-359 155 155 155 253 253 253 F F F 18 360-379 159 159 159 259 259 259 F F F 19 380-399 159 159 159 259 259 259 F F F 20 400-419 160 160 160 260 260 260 F F F 21 420-439 155 155 155 253 253 253 F F F 22 440-459 155 155 155 253 253 253 F F F 23 460-479 160 160 160 260 260 260 F F F 24 480-499 156 156 156 254 254 254 F F F 25 500-519 160 160 160 260 260 260 F F F 26 520-539 158 158 158 257 257 257 F F F 27 540-559 158 158 158 257 257 257 F F F 28 560-579 158 158 158 257 257 257 F F F 29 580-599 154 154 154 252 252 252 F F F Max gen: 160, Max flush: 260 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.499752096Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.544407921Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.545146211Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.545199722Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.545218306Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.545242171Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836} {"msg":"Crucible 15b8595d-7dc2-4ee1-857a-a4b953ed7871 has session id: 3f3d8d69-bda2-40a4-9208-14da89731c02","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.545844591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Upstairs opts: Upstairs UUID: 15b8595d-7dc2-4ee1-857a-a4b953ed7871, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.545898393Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Crucible stats registered with UUID: 15b8595d-7dc2-4ee1-857a-a4b953ed7871","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.545921738Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:53.545937882Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.546360653Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.54639487Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836} {"msg":"15b8595d-7dc2-4ee1-857a-a4b953ed7871 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.546461535Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.546478798Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"0","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.546506554Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"0","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.546520637Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"1","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.54653435Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"1","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.546548153Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"2","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.546562196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"2","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.54658038Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.546604185Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.54662805Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"ds_connection connected from Ok(127.0.0.1:41930)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.546652776Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"ds_connection connected from Ok(127.0.0.1:34495)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.546888407Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"ds_connection connected from Ok(127.0.0.1:35563)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.546924744Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.547571554Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"0","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.54759979Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"2","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.548029873Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"1","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.548255182Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"0","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.548283988Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"2","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.548576401Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"1","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.548866633Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.548981368Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"[0]R flush_numbers[0..12]: [254, 260, 260, 260, 260, 254, 256, 260, 256, 256, 257, 255]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549116907Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"[0]R generation[0..12]: [156, 160, 160, 160, 160, 156, 157, 160, 157, 157, 158, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549140722Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549156546Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"[1]R flush_numbers[0..12]: [254, 260, 260, 260, 260, 254, 256, 260, 256, 256, 257, 255]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549170569Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"[1]R generation[0..12]: [156, 160, 160, 160, 160, 156, 157, 160, 157, 157, 158, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549184272Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549198255Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"[2]R flush_numbers[0..12]: [254, 260, 260, 260, 260, 254, 256, 260, 256, 256, 257, 255]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549211668Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"[2]R generation[0..12]: [156, 160, 160, 160, 160, 156, 157, 160, 157, 157, 158, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549224841Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549238294Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Max found gen is 161","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549250736Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Generation requested: 162 >= found:161","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549263449Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO "2025-03-17T22:30:53.549275692Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549308859Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"":"downstairs","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549322122Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549333814Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"0","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549348778Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"1","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.54936198Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"client":"2","":"downstairs","session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549374733Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549386595Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"15b8595d-7dc2-4ee1-857a-a4b953ed7871 is now active with session: 3f3d8d69-bda2-40a4-9208-14da89731c02","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549398938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549410441Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836,"session_id":"3f3d8d69-bda2-40a4-9208-14da89731c02"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549422353Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836} {"msg":"Activated sub_volume 15b8595d-7dc2-4ee1-857a-a4b953ed7871","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:53.549433106Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 390 len 8192 data: 15 10 02/30 Flush 03/30 Write block 363 len 36864 data: 15 12 13 8 8 7 6 8 8 04/30 Read block 357 len 24576 05/30 Write block 61 len 16384 data: 12 14 12 14 06/30 Write block 115 len 12288 data: 17 16 15 07/30 Read block 176 len 8192 08/30 Write block 362 len 20480 data: 14 16 13 14 9 09/30 Read block 305 len 40960 10/30 Read block 381 len 20480 11/30 Write block 102 len 36864 data: 9 8 9 8 8 8 10 8 8 12/30 Flush 13/30 Write block 330 len 36864 data: 9 14 15 15 17 18 15 11 11 14/30 Read block 521 len 4096 15/30 Flush 16/30 Write block 131 len 32768 data: 13 13 13 12 12 12 9 13 17/30 Read block 197 len 4096 18/30 Read block 239 len 4096 19/30 Read block 382 len 24576 20/30 Write block 586 len 8192 data: 12 13 21/30 Read block 238 len 32768 22/30 Read block 447 len 36864 23/30 Read block 258 len 36864 24/30 Write block 368 len 40960 data: 8 7 9 9 7 8 8 11 8 8 25/30 Read block 426 len 36864 26/30 Write block 512 len 12288 data: 9 9 10 27/30 Read block 385 len 32768 28/30 Write block 150 len 28672 data: 15 19 18 18 21 14 14 29/30 Write block 216 len 8192 data: 9 10 30/30 Write block 492 len 24576 data: 16 14 12 13 13 12 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 8 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 7 Done Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 3 8 Done 13 12 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:53.569834207Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1836} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 160 160 160 260 260 260 F F F 2 040-059 160 160 160 260 260 260 F F F 3 060-079 162 162 162 262 262 262 F F F 4 080-099 160 160 160 260 260 260 F F F 5 100-119 162 162 162 262 262 262 F F F 6 120-139 157 157 157 256 256 256 T T T 7 140-159 160 160 160 260 260 260 T T T 8 160-179 157 157 157 256 256 256 F F F 9 180-199 157 157 157 256 256 256 F F F 10 200-219 158 158 158 257 257 257 T T T 11 220-239 156 156 156 255 255 255 F F F 12 240-259 158 158 158 258 258 258 F F F 13 260-279 158 158 158 258 258 258 F F F 14 280-299 159 159 159 259 259 259 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 162 162 162 263 263 263 F F F 17 340-359 155 155 155 253 253 253 F F F 18 360-379 162 162 162 262 262 262 T T T 19 380-399 162 162 162 261 261 261 F F F 20 400-419 160 160 160 260 260 260 F F F 21 420-439 155 155 155 253 253 253 F F F 22 440-459 155 155 155 253 253 253 F F F 23 460-479 160 160 160 260 260 260 F F F 24 480-499 156 156 156 254 254 254 T T T 25 500-519 160 160 160 260 260 260 T T T 26 520-539 158 158 158 257 257 257 F F F 27 540-559 158 158 158 257 257 257 F F F 28 560-579 158 158 158 257 257 257 F F F 29 580-599 154 154 154 252 252 252 T T T Max gen: 162, Max flush: 263 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 160 160 160 260 260 260 F F F 2 040-059 160 160 160 260 260 260 F F F 3 060-079 162 162 162 262 262 262 F F F 4 080-099 160 160 160 260 260 260 F F F 5 100-119 162 162 162 262 262 262 F F F 6 120-139 163 163 163 264 264 264 F F F 7 140-159 163 163 163 264 264 264 F F F 8 160-179 157 157 157 256 256 256 F F F 9 180-199 157 157 157 256 256 256 F F F 10 200-219 163 163 163 264 264 264 F F F 11 220-239 156 156 156 255 255 255 F F F 12 240-259 158 158 158 258 258 258 F F F 13 260-279 158 158 158 258 258 258 F F F 14 280-299 159 159 159 259 259 259 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 162 162 162 263 263 263 F F F 17 340-359 155 155 155 253 253 253 F F F 18 360-379 163 163 163 264 264 264 F F F 19 380-399 162 162 162 261 261 261 F F F 20 400-419 160 160 160 260 260 260 F F F 21 420-439 155 155 155 253 253 253 F F F 22 440-459 155 155 155 253 253 253 F F F 23 460-479 160 160 160 260 260 260 F F F 24 480-499 163 163 163 264 264 264 F F F 25 500-519 163 163 163 264 264 264 F F F 26 520-539 158 158 158 257 257 257 F F F 27 540-559 158 158 158 257 257 257 F F F 28 560-579 158 158 158 257 257 257 F F F 29 580-599 163 163 163 264 264 264 F F F Max gen: 163, Max flush: 264 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.822001377Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.866019255Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.866816737Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.866867938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.866885452Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.866904076Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844} {"msg":"Crucible cee182a4-23ae-40c8-8c2f-b0f35b35ceda has session id: bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.867506506Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Upstairs opts: Upstairs UUID: cee182a4-23ae-40c8-8c2f-b0f35b35ceda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.867550785Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Crucible stats registered with UUID: cee182a4-23ae-40c8-8c2f-b0f35b35ceda","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.86757364Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:57.867594205Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868020417Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868054794Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844} {"msg":"cee182a4-23ae-40c8-8c2f-b0f35b35ceda active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868104545Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868121839Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"0","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868149565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"0","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868164068Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"1","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868178241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"1","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868191003Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"2","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868204706Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"2","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868217559Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868231072Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868252517Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"ds_connection connected from Ok(127.0.0.1:54424)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868400599Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:57.868445388Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"ds_connection connected from Ok(127.0.0.1:49995)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868478536Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:57.868514413Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:30:57.868647232Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.868757716Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.86924282Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"0","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.869271827Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"1","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.869964186Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"1","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.870004755Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"0","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.870602494Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:30:57.870628329Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.869584725Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"ds_connection connected from Ok(127.0.0.1:33546)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.869739398Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.870978016Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"2","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.871455289Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"2","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"[0]R flush_numbers[0..12]: [254, 260, 260, 262, 260, 262, 264, 264, 256, 256, 264, 255]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872059149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"[0]R generation[0..12]: [156, 160, 160, 162, 160, 162, 163, 163, 157, 157, 163, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872089525Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872113941Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"[1]R flush_numbers[0..12]: [254, 260, 260, 262, 260, 262, 264, 264, 256, 256, 264, 255]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872132065Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"[1]R generation[0..12]: [156, 160, 160, 162, 160, 162, 163, 163, 157, 157, 163, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872148868Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872166282Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"[2]R flush_numbers[0..12]: [254, 260, 260, 262, 260, 262, 264, 264, 256, 256, 264, 255]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872183316Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"[2]R generation[0..12]: [156, 160, 160, 162, 160, 162, 163, 163, 157, 157, 163, 156]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.87220112Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872218443Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872234357Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.87225052Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872266223Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872281837Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"":"downstairs","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.87229768Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872312153Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"0","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:31:07.872328497Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"1","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872350502Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"client":"2","":"downstairs","session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872368806Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872383619Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"cee182a4-23ae-40c8-8c2f-b0f35b35ceda is now active with session: bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872399122Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872413595Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844,"session_id":"bea1a5ca-7f84-4b99-8bc8-6aab32ed5b65"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872428369Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844} {"msg":"Activated sub_volume cee182a4-23ae-40c8-8c2f-b0f35b35ceda","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:07.872442112Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 163 len 16384 data: 14 17 20 18 02/30 Read block 135 len 20480 03/30 Read block 133 len 4096 04/30 Flush 05/30 Write block 355 len 24576 data: 8 10 12 16 16 17 06/30 Read block 514 len 32768 07/30 Read block 206 len 12288 08/30 Write block 376 len 40960 data: 9 9 10 13 12 13 12 12 11 13 09/30 Read block 459 len 24576 10/30 Write block 284 len 32768 data: 14 16 15 14 15 14 16 13 11/30 Write block 30 len 28672 data: 10 10 8 9 10 12 11 12/30 Write block 280 len 40960 data: 12 12 12 13 15 17 16 15 16 15 13/30 Read block 66 len 36864 14/30 Flush 15/30 Write block 440 len 24576 data: 15 12 15 14 12 13 16/30 Write block 343 len 12288 data: 11 12 10 17/30 Flush 18/30 Write block 69 len 12288 data: 14 15 15 19/30 Write block 501 len 12288 data: 17 16 16 20/30 Write block 258 len 28672 data: 13 12 13 10 9 13 14 21/30 Write block 579 len 20480 data: 15 11 12 12 12 22/30 Write block 466 len 8192 data: 9 15 23/30 Read block 139 len 32768 24/30 Write block 225 len 20480 data: 8 9 11 14 13 25/30 Read block 266 len 36864 26/30 Write block 332 len 24576 data: 16 16 18 19 16 12 27/30 Read block 438 len 24576 28/30 Write block 45 len 4096 data: 16 29/30 Read block 465 len 12288 30/30 Read block 370 len 8192 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:07.892520934Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1844} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 164 164 164 266 266 266 F F F 2 040-059 160 160 160 260 260 260 T T T 3 060-079 162 162 162 262 262 262 T T T 4 080-099 160 160 160 260 260 260 F F F 5 100-119 162 162 162 262 262 262 F F F 6 120-139 163 163 163 264 264 264 F F F 7 140-159 163 163 163 264 264 264 F F F 8 160-179 164 164 164 265 265 265 F F F 9 180-199 157 157 157 256 256 256 F F F 10 200-219 163 163 163 264 264 264 F F F 11 220-239 156 156 156 255 255 255 T T T 12 240-259 158 158 158 258 258 258 T T T 13 260-279 158 158 158 258 258 258 T T T 14 280-299 164 164 164 266 266 266 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 162 162 162 263 263 263 T T T 17 340-359 164 164 164 267 267 267 F F F 18 360-379 164 164 164 266 266 266 F F F 19 380-399 164 164 164 266 266 266 F F F 20 400-419 160 160 160 260 260 260 F F F 21 420-439 155 155 155 253 253 253 F F F 22 440-459 164 164 164 267 267 267 F F F 23 460-479 160 160 160 260 260 260 T T T 24 480-499 163 163 163 264 264 264 F F F 25 500-519 163 163 163 264 264 264 T T T 26 520-539 158 158 158 257 257 257 F F F 27 540-559 158 158 158 257 257 257 F F F 28 560-579 158 158 158 257 257 257 T T T 29 580-599 163 163 163 264 264 264 T T T Max gen: 164, Max flush: 267 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 164 164 164 266 266 266 F F F 2 040-059 165 165 165 268 268 268 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 160 160 160 260 260 260 F F F 5 100-119 162 162 162 262 262 262 F F F 6 120-139 163 163 163 264 264 264 F F F 7 140-159 163 163 163 264 264 264 F F F 8 160-179 164 164 164 265 265 265 F F F 9 180-199 157 157 157 256 256 256 F F F 10 200-219 163 163 163 264 264 264 F F F 11 220-239 165 165 165 268 268 268 F F F 12 240-259 165 165 165 268 268 268 F F F 13 260-279 165 165 165 268 268 268 F F F 14 280-299 164 164 164 266 266 266 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 165 165 165 268 268 268 F F F 17 340-359 164 164 164 267 267 267 F F F 18 360-379 164 164 164 266 266 266 F F F 19 380-399 164 164 164 266 266 266 F F F 20 400-419 160 160 160 260 260 260 F F F 21 420-439 155 155 155 253 253 253 F F F 22 440-459 164 164 164 267 267 267 F F F 23 460-479 165 165 165 268 268 268 F F F 24 480-499 163 163 163 264 264 264 F F F 25 500-519 165 165 165 268 268 268 F F F 26 520-539 158 158 158 257 257 257 F F F 27 540-559 158 158 158 257 257 257 F F F 28 560-579 165 165 165 268 268 268 F F F 29 580-599 165 165 165 268 268 268 F F F Max gen: 165, Max flush: 268 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.161451542Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.205529843Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.206236126Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.206276474Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.206291918Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.206306101Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852} {"msg":"Crucible fd3f5768-5948-4858-882f-511a0452a430 has session id: cef8e06c-f89f-4572-8765-2161bb6f8671","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.206915472Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Upstairs opts: Upstairs UUID: fd3f5768-5948-4858-882f-511a0452a430, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.206959732Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Crucible stats registered with UUID: fd3f5768-5948-4858-882f-511a0452a430","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.206983247Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:22.207004102Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207449898Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207480544Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852} {"msg":"fd3f5768-5948-4858-882f-511a0452a430 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207502189Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207521783Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"0","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207563682Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"0","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207585877Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"1","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207675936Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"1","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207716505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"2","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207744991Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"2","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207772347Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207800033Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.20782954Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"ds_connection connected from Ok(127.0.0.1:52657)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207860086Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"ds_connection connected from Ok(127.0.0.1:50110)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207887632Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"ds_connection connected from Ok(127.0.0.1:43455)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.207910227Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.208365155Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"0","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.208399583Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"1","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.208963934Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"1","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.209107745Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"0","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.209581098Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.209665856Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.209956539Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"2","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.210436822Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"2","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"[0]R flush_numbers[0..12]: [254, 266, 268, 268, 260, 262, 264, 264, 265, 256, 264, 268]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.210939431Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"[0]R generation[0..12]: [156, 164, 165, 165, 160, 162, 163, 163, 164, 157, 163, 165]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.210963766Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.210978879Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"[1]R flush_numbers[0..12]: [254, 266, 268, 268, 260, 262, 264, 264, 265, 256, 264, 268]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.210993042Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"[1]R generation[0..12]: [156, 164, 165, 165, 160, 162, 163, 163, 164, 157, 163, 165]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211008215Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211025939Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"[2]R flush_numbers[0..12]: [254, 266, 268, 268, 260, 262, 264, 264, 265, 256, 264, 268]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211042623Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"[2]R generation[0..12]: [156, 164, 165, 165, 160, 162, 163, 163, 164, 157, 163, 165]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211059326Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.21107611Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211095384Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211111708Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211135763Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211151476Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"":"downstairs","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.21116769Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211182063Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"0","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211198066Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"1","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.21121367Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"client":"2","":"downstairs","session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211229453Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211244276Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"fd3f5768-5948-4858-882f-511a0452a430 is now active with session: cef8e06c-f89f-4572-8765-2161bb6f8671","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.21125981Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211274113Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852,"session_id":"cef8e06c-f89f-4572-8765-2161bb6f8671"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:22.211288846Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852} {"msg":"Activated sub_volume fd3f5768-5948-4858-882f-511a0452a430","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-17T22:31:22.211302239Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 419 len 4096 data: 10 02/30 Write block 279 len 36864 data: 9 13 13 13 14 16 18 17 16 03/30 Write block 549 len 32768 data: 14 13 12 9 10 11 12 13 04/30 Write block 135 len 4096 data: 13 05/30 Read block 190 len 12288 06/30 Write block 328 len 40960 data: 10 9 10 15 17 17 19 20 17 13 07/30 Write block 442 len 8192 data: 16 15 08/30 Write block 236 len 36864 data: 13 11 11 10 11 10 8 7 7 09/30 Read block 49 len 36864 10/30 Write block 175 len 16384 data: 9 12 12 14 11/30 Read block 176 len 28672 12/30 Read block 494 len 28672 13/30 Read block 541 len 8192 14/30 Write block 341 len 4096 data: 10 15/30 Flush 16/30 Write block 554 len 32768 data: 12 13 14 13 15 15 18 17 17/30 Write block 116 len 4096 data: 17 18/30 Read block 280 len 28672 19/30 Flush 20/30 Write block 208 len 16384 data: 13 16 15 16 21/30 Read block 4 len 8192 22/30 Read block 407 len 24576 23/30 Read block 405 len 40960 24/30 Read block 244 len 4096 25/30 Write block 410 len 8192 data: 14 14 26/30 Read block 90 len 8192 27/30 Write block 392 len 12288 data: 10 14 13 28/30 Read block 539 len 24576 29/30 Read block 102 len 40960 30/30 Read block 310 len 40960 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:22.230599253Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1852} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 164 164 164 266 266 266 F F F 2 040-059 165 165 165 268 268 268 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 160 160 160 260 260 260 F F F 5 100-119 166 166 166 270 270 270 F F F 6 120-139 166 166 166 269 269 269 F F F 7 140-159 163 163 163 264 264 264 F F F 8 160-179 166 166 166 269 269 269 F F F 9 180-199 157 157 157 256 256 256 F F F 10 200-219 163 163 163 264 264 264 T T T 11 220-239 166 166 166 269 269 269 F F F 12 240-259 166 166 166 269 269 269 F F F 13 260-279 166 166 166 269 269 269 F F F 14 280-299 166 166 166 269 269 269 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 166 166 166 269 269 269 F F F 17 340-359 166 166 166 269 269 269 F F F 18 360-379 164 164 164 266 266 266 F F F 19 380-399 164 164 164 266 266 266 T T T 20 400-419 166 166 166 269 269 269 T T T 21 420-439 155 155 155 253 253 253 F F F 22 440-459 166 166 166 269 269 269 F F F 23 460-479 165 165 165 268 268 268 F F F 24 480-499 163 163 163 264 264 264 F F F 25 500-519 165 165 165 268 268 268 F F F 26 520-539 158 158 158 257 257 257 F F F 27 540-559 166 166 166 270 270 270 F F F 28 560-579 166 166 166 270 270 270 F F F 29 580-599 165 165 165 268 268 268 F F F Max gen: 166, Max flush: 270 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 164 164 164 266 266 266 F F F 2 040-059 165 165 165 268 268 268 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 160 160 160 260 260 260 F F F 5 100-119 166 166 166 270 270 270 F F F 6 120-139 166 166 166 269 269 269 F F F 7 140-159 163 163 163 264 264 264 F F F 8 160-179 166 166 166 269 269 269 F F F 9 180-199 157 157 157 256 256 256 F F F 10 200-219 167 167 167 271 271 271 F F F 11 220-239 166 166 166 269 269 269 F F F 12 240-259 166 166 166 269 269 269 F F F 13 260-279 166 166 166 269 269 269 F F F 14 280-299 166 166 166 269 269 269 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 166 166 166 269 269 269 F F F 17 340-359 166 166 166 269 269 269 F F F 18 360-379 164 164 164 266 266 266 F F F 19 380-399 167 167 167 271 271 271 F F F 20 400-419 167 167 167 271 271 271 F F F 21 420-439 155 155 155 253 253 253 F F F 22 440-459 166 166 166 269 269 269 F F F 23 460-479 165 165 165 268 268 268 F F F 24 480-499 163 163 163 264 264 264 F F F 25 500-519 165 165 165 268 268 268 F F F 26 520-539 158 158 158 257 257 257 F F F 27 540-559 166 166 166 270 270 270 F F F 28 560-579 166 166 166 270 270 270 F F F 29 580-599 165 165 165 268 268 268 F F F Max gen: 167, Max flush: 271 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.457739293Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.501916195Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.502571977Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.502622177Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.502639781Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.502658645Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860} {"msg":"Crucible 4e6d6027-2074-4de4-9f5d-7648ab2bff8f has session id: 2db9ff21-c128-498d-8fc7-50e371785f6c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503251743Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Upstairs opts: Upstairs UUID: 4e6d6027-2074-4de4-9f5d-7648ab2bff8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503299234Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Crucible stats registered with UUID: 4e6d6027-2074-4de4-9f5d-7648ab2bff8f","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503323219Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:36.503342023Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503726486Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503752581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860} {"msg":"4e6d6027-2074-4de4-9f5d-7648ab2bff8f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503821866Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.50383951Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"0","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503868006Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"0","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503881899Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"1","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503896052Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"1","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503909115Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"2","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503922758Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"2","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503955005Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.503970948Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.504021559Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:53428)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.504060038Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:41125)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.504087073Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47377)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.504111959Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.504744365Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"0","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.504772521Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"2","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.505407238Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"2","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.505574634Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"0","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.505903515Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"1","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.506067571Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.506153779Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.506423878Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"1","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"[0]R flush_numbers[0..12]: [254, 266, 268, 268, 260, 270, 269, 264, 269, 256, 271, 269]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.506948861Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"[0]R generation[0..12]: [156, 164, 165, 165, 160, 166, 166, 163, 166, 157, 167, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.506968375Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.506982698Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"[1]R flush_numbers[0..12]: [254, 266, 268, 268, 260, 270, 269, 264, 269, 256, 271, 269]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507000752Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"[1]R generation[0..12]: [156, 164, 165, 165, 160, 166, 166, 163, 166, 157, 167, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507015065Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507028548Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"[2]R flush_numbers[0..12]: [254, 266, 268, 268, 260, 270, 269, 264, 269, 256, 271, 269]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507041991Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"[2]R generation[0..12]: [156, 164, 165, 165, 160, 166, 166, 163, 166, 157, 167, 166]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507055394Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507068957Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507081419Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507094122Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507106535Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":Wait for a query_work_queue command to finish before sending IO 1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507125459Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"":"downstairs","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507138282Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507149754Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"0","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507162557Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"1","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.50717521Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"client":"2","":"downstairs","session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507188113Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507201685Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"4e6d6027-2074-4de4-9f5d-7648ab2bff8f is now active with session: 2db9ff21-c128-498d-8fc7-50e371785f6c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507214208Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507225841Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860,"session_id":"2db9ff21-c128-498d-8fc7-50e371785f6c"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507237693Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860} {"msg":"Activated sub_volume 4e6d6027-2074-4de4-9f5d-7648ab2bff8f","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:36.507248466Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 219 len 28672 data: 9 8 8 7 8 11 9 02/30 Write block 154 len 8192 data: 22 15 03/30 Read block 325 len 24576 04/30 Write block 288 len 12288 data: 17 16 17 05/30 Write block 457 len 32768 data: 12 11 11 13 15 14 15 15 06/30 Flush 07/30 Write block 561 len 20480 data: 18 19 19 18 17 08/30 Read block 556 len 4096 09/30 Read block 260 len 16384 10/30 Read block 22 len 8192 11/30 Read block 498 len 4096 12/30 Write block 416 len 4096 data: 9 13/30 Write block 168 len 12288 data: 13 16 13 14/30 Write block 223 len 40960 data: 9 12 10 10 12 15 14 18 14 14 15/30 Write block 429 len 8192 data: 13 15 16/30 Read block 370 len 16384 17/30 Read block 591 len 32768 18/30 Read block 573 len 8192 19/30 Read block 18 len 24576 20/30 Read block 55 len 4096 21/30 Read block 550 len 28672 22/30 Read block 100 len 36864 23/30 Write block 48 len 8192 data: 18 18 24/30 Write block 388 len 32768 data: 13 13 16 11 11 15 14 12 25/30 Write block 179 len 24576 data: 17 16 16 17 17 21 26/30 Read block 354 len 36864 27/30 Write block 122 len 28672 data: 12 12 8 6 6 7 9 28/30 Write block 548 len 8192 data: 14 15 29/30 Write block 254 len 24576 data: 13 12 11 12 14 13 30/30 Write block 206 len 8192 data: 14 12 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 5 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 7 Sent Done Sent false 1027 Acked Write 2 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 3 4 11 Done 20 21 20 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:36.525472968Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1860} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 164 164 164 266 266 266 F F F 2 040-059 165 165 165 268 268 268 T T T 3 060-079 165 165 165 268 268 268 F F F 4 080-099 160 160 160 260 260 260 F F F 5 100-119 166 166 166 270 270 270 F F F 6 120-139 166 166 166 269 269 269 T T T 7 140-159 168 168 168 272 272 272 F F F 8 160-179 166 166 166 269 269 269 T T T 9 180-199 157 157 157 256 256 256 T T T 10 200-219 168 168 168 272 272 272 T T T 11 220-239 168 168 168 272 272 272 T T T 12 240-259 166 166 166 269 269 269 T T T 13 260-279 166 166 166 269 269 269 F F F 14 280-299 168 168 168 272 272 272 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 166 166 166 269 269 269 F F F 17 340-359 166 166 166 269 269 269 F F F 18 360-379 164 164 164 266 266 266 F F F 19 380-399 167 167 167 271 271 271 T T T 20 400-419 167 167 167 271 271 271 T T T 21 420-439 155 155 155 253 253 253 T T T 22 440-459 168 168 168 272 272 272 F F F 23 460-479 168 168 168 272 272 272 F F F 24 480-499 163 163 163 264 264 264 F F F 25 500-519 165 165 165 268 268 268 F F F 26 520-539 158 158 158 257 257 257 F F F 27 540-559 166 166 166 270 270 270 T T T 28 560-579 166 166 166 270 270 270 T T T 29 580-599 165 165 165 268 268 268 F F F Max gen: 168, Max flush: 272 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 164 164 164 266 266 266 F F F 2 040-059 169 169 169 273 273 273 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 160 160 160 260 260 260 F F F 5 100-119 166 166 166 270 270 270 F F F 6 120-139 169 169 169 273 273 273 F F F 7 140-159 168 168 168 272 272 272 F F F 8 160-179 169 169 169 273 273 273 F F F 9 180-199 169 169 169 273 273 273 F F F 10 200-219 169 169 169 273 273 273 F F F 11 220-239 169 169 169 273 273 273 F F F 12 240-259 169 169 169 273 273 273 F F F 13 260-279 166 166 166 269 269 269 F F F 14 280-299 168 168 168 272 272 272 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 166 166 166 269 269 269 F F F 17 340-359 166 166 166 269 269 269 F F F 18 360-379 164 164 164 266 266 266 F F F 19 380-399 169 169 169 273 273 273 F F F 20 400-419 169 169 169 273 273 273 F F F 21 420-439 169 169 169 273 273 273 F F F 22 440-459 168 168 168 272 272 272 F F F 23 460-479 168 168 168 272 272 272 F F F 24 480-499 163 163 163 264 264 264 F F F 25 500-519 165 165 165 268 268 268 F F F 26 520-539 158 158 158 257 257 257 F F F 27 540-559 169 169 169 273 273 273 F F F 28 560-579 169 169 169 273 273 273 F F F 29 580-599 165 165 165 268 268 268 F F F Max gen: 169, Max flush: 273 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.801284Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:40.843511081Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.843863998Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.874920279Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.875521838Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.875555616Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.875569579Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.875583412Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868} {"msg":"Crucible d138a58c-b818-4c05-bf4a-f1d96c26b2c6 has session id: 0b419145-819c-41e4-a95f-0109ea833fa9","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876202115Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Upstairs opts: Upstairs UUID: d138a58c-b818-4c05-bf4a-f1d96c26b2c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876237383Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Crucible stats registered with UUID: d138a58c-b818-4c05-bf4a-f1d96c26b2c6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876252196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:40.876264769Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876636329Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876654293Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868} {"msg":"d138a58c-b818-4c05-bf4a-f1d96c26b2c6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876719097Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.87673537Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"0","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876763076Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"0","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876776809Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"1","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876790542Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"1","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876803295Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"2","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876816978Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"2","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876830011Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876843363Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876856426Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:48947)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876958408Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:57050)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.876999287Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:47330)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.877179436Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.877653728Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"1","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.877855002Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"0","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.87794068Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"2","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.878368532Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"1","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.878444989Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"0","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.878557043Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"2","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879025284Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879059692Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"[0]R flush_numbers[0..12]: [254, 266, 273, 268, 260, 270, 273, 272, 273, 273, 273, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879083037Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"[0]R generation[0..12]: [156, 164, 169, 165, 160, 166, 169, 168, 169, 169, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879108492Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879133137Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"[1]R flush_numbers[0..12]: [254, 266, 273, 268, 260, 270, 273, 272, 273, 273, 273, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879158113Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"[1]R generation[0..12]: [156, 164, 169, 165, 160, 166, 169, 168, 169, 169, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879184579Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879209904Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"[2]R flush_numbers[0..12]: [254, 266, 273, 268, 260, 270, 273, 272, 273, 273, 273, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.87923654Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"[2]R generation[0..12]: [156, 164, 169, 165, 160, 166, 169, 168, 169, 169, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879263306Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879290962Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879316937Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879343473Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Next flush: 274","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:31:40.879369428Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879397645Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"":"downstairs","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.87942227Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879445705Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"0","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.87947016Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"1","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879495766Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"client":"2","":"downstairs","session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879521511Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879546027Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"d138a58c-b818-4c05-bf4a-f1d96c26b2c6 is now active with session: 0b419145-819c-41e4-a95f-0109ea833fa9","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879571242Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-17T22:31:40.879594767Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868,"session_id":"0b419145-819c-41e4-a95f-0109ea833fa9"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879622503Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868} {"msg":"Activated sub_volume d138a58c-b818-4c05-bf4a-f1d96c26b2c6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:40.879638357Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 189 len 16384 data: 11 9 8 8 02/30 Flush 03/30 Write block 373 len 28672 data: 9 9 12 10 10 11 14 04/30 Read block 551 len 20480 05/30 Read block 410 len 32768 06/30 Write block 536 len 36864 data: 12 14 13 14 14 13 15 17 17 07/30 Read block 158 len 36864 08/30 Read block 282 len 24576 09/30 Write block 192 len 20480 data: 9 9 13 12 12 10/30 Flush 11/30 Write block 275 len 16384 data: 10 12 9 10 12/30 Read block 498 len 28672 13/30 Write block 556 len 16384 data: 15 14 16 16 14/30 Read block 451 len 36864 15/30 Read block 182 len 8192 16/30 Read block 532 len 12288 17/30 Read block 447 len 40960 18/30 Write block 84 len 16384 data: 9 10 10 9 19/30 Read block 263 len 4096 20/30 Write block 84 len 20480 data: 10 11 11 10 10 21/30 Read block 485 len 12288 22/30 Read block 466 len 12288 23/30 Write block 579 len 28672 data: 16 12 13 13 13 15 15 24/30 Write block 250 len 20480 data: 12 13 13 12 14 25/30 Write block 333 len 24576 data: 18 20 21 18 14 12 26/30 Read block 161 len 4096 27/30 Read block 341 len 4096 28/30 Write block 179 len 24576 data: 18 17 17 18 18 22 29/30 Read block 48 len 4096 30/30 Read block 253 len 40960 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 4 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 19 19 20 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:40.900227929Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1868} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 164 164 164 266 266 266 F F F 2 040-059 169 169 169 273 273 273 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 160 160 160 260 260 260 T T T 5 100-119 166 166 166 270 270 270 F F F 6 120-139 169 169 169 273 273 273 F F F 7 140-159 168 168 168 272 272 272 F F F 8 160-179 169 169 169 273 273 273 T T T 9 180-199 170 170 170 275 275 275 T T T 10 200-219 169 169 169 273 273 273 F F F 11 220-239 169 169 169 273 273 273 F F F 12 240-259 169 169 169 273 273 273 T T T 13 260-279 166 166 166 269 269 269 T T T 14 280-299 168 168 168 272 272 272 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 166 166 166 269 269 269 T T T 17 340-359 166 166 166 269 269 269 F F F 18 360-379 170 170 170 275 275 275 F F F 19 380-399 169 169 169 273 273 273 F F F 20 400-419 169 169 169 273 273 273 F F F 21 420-439 169 169 169 273 273 273 F F F 22 440-459 168 168 168 272 272 272 F F F 23 460-479 168 168 168 272 272 272 F F F 24 480-499 163 163 163 264 264 264 F F F 25 500-519 165 165 165 268 268 268 F F F 26 520-539 170 170 170 275 275 275 F F F 27 540-559 170 170 170 275 275 275 T T T 28 560-579 169 169 169 273 273 273 T T T 29 580-599 165 165 165 268 268 268 T T T Max gen: 170, Max flush: 275 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 164 164 164 266 266 266 F F F 2 040-059 169 169 169 273 273 273 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 171 171 171 276 276 276 F F F 5 100-119 166 166 166 270 270 270 F F F 6 120-139 169 169 169 273 273 273 F F F 7 140-159 168 168 168 272 272 272 F F F 8 160-179 171 171 171 276 276 276 F F F 9 180-199 171 171 171 276 276 276 F F F 10 200-219 169 169 169 273 273 273 F F F 11 220-239 169 169 169 273 273 273 F F F 12 240-259 171 171 171 276 276 276 F F F 13 260-279 171 171 171 276 276 276 F F F 14 280-299 168 168 168 272 272 272 F F F 15 300-319 156 156 156 255 255 255 F F F 16 320-339 171 171 171 276 276 276 F F F 17 340-359 166 166 166 269 269 269 F F F 18 360-379 170 170 170 275 275 275 F F F 19 380-399 169 169 169 273 273 273 F F F 20 400-419 169 169 169 273 273 273 F F F 21 420-439 169 169 169 273 273 273 F F F 22 440-459 168 168 168 272 272 272 F F F 23 460-479 168 168 168 272 272 272 F F F 24 480-499 163 163 163 264 264 264 F F F 25 500-519 165 165 165 268 268 268 F F F 26 520-539 170 170 170 275 275 275 F F F 27 540-559 171 171 171 276 276 276 F F F 28 560-579 171 171 171 276 276 276 F F F 29 580-599 171 171 171 276 276 276 F F F Max gen: 171, Max flush: 276 Begin loop 86 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.188410511Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.233443698Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.234155521Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.234208033Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.234230758Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.234249482Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876} {"msg":"Crucible 6f9a619a-b4dc-46a0-a1a6-ab06a3e13458 has session id: 44cef201-a0d9-4743-ba6e-57b1f8051eb8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.234899192Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Upstairs opts: Upstairs UUID: 6f9a619a-b4dc-46a0-a1a6-ab06a3e13458, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.234947472Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Crucible stats registered with UUID: 6f9a619a-b4dc-46a0-a1a6-ab06a3e13458","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.234974038Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:45.235000234Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235397489Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235421305Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876} {"msg":"6f9a619a-b4dc-46a0-a1a6-ab06a3e13458 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235489689Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235506423Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"0","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235534509Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"0","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235548882Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"1","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235563075Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"1","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235579889Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"2","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235594562Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"2","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235607445Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235621328Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.23572438Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:62629)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235766509Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:63051)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235796675Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:43732)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.235825152Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.236523452Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"1","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.236553929Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"0","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.237229815Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"1","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.237260951Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"0","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.237407833Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"2","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.23794793Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.237983747Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238012714Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"2","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"[0]R flush_numbers[0..12]: [254, 266, 273, 268, 276, 270, 273, 272, 276, 276, 273, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238542998Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"[0]R generation[0..12]: [156, 164, 169, 165, 171, 166, 169, 168, 171, 171, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238571494Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238586848Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"[1]R flush_numbers[0..12]: [254, 266, 273, 268, 276, 270, 273, 272, 276, 276, 273, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238610613Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"[1]R generation[0..12]: [156, 164, 169, 165, 171, 166, 169, 168, 171, 171, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238635698Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238663794Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"[2]R flush_numbers[0..12]: [254, 266, 273, 268, 276, 270, 273, 272, 276, 276, 273, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.23868981Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"[2]R generation[0..12]: [156, 164, 169, 165, 171, 166, 169, 168, 171, 171, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238717036Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238740991Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238755194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238776549Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"Wait for a query_work_queue command to finish before sending IO ":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238806835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238836591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"":"downstairs","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238853465Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238865198Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"0","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238880691Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"1","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238903756Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"client":"2","":"downstairs","session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238928541Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238950416Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"6f9a619a-b4dc-46a0-a1a6-ab06a3e13458 is now active with session: 44cef201-a0d9-4743-ba6e-57b1f8051eb8","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238963919Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238977232Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876,"session_id":"44cef201-a0d9-4743-ba6e-57b1f8051eb8"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.238998346Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876} {"msg":"Activated sub_volume 6f9a619a-b4dc-46a0-a1a6-ab06a3e13458","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:45.239018901Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 112 len 12288 data: 10 12 14 02/30 Write block 337 len 28672 data: 15 13 13 12 11 13 12 03/30 Read block 21 len 4096 04/30 Write block 541 len 24576 data: 14 16 18 18 15 17 05/30 Read block 565 len 40960 06/30 Read block 382 len 28672 07/30 Write block 242 len 28672 data: 9 8 8 8 12 13 12 08/30 Write block 245 len 12288 data: 9 13 14 09/30 Read block 124 len 32768 10/30 Read block 559 len 20480 11/30 Read block 190 len 4096 12/30 Read block 522 len 20480 13/30 Flush 14/30 Write block 351 len 20480 data: 9 8 9 6 9 15/30 Read block 206 len 12288 16/30 Read block 527 len 28672 17/30 Read block 331 len 24576 18/30 Write block 342 len 16384 data: 14 13 13 11 19/30 Write block 416 len 12288 data: 10 8 9 20/30 Write block 297 len 28672 data: 14 13 11 13 13 15 15 21/30 Read block 408 len 36864 22/30 Read block 262 len 20480 23/30 Read block 372 len 8192 24/30 Write block 559 len 12288 data: 17 19 19 25/30 Read block 335 len 12288 26/30 Read block 214 len 40960 27/30 Write block 534 len 4096 data: 14 28/30 Read block 582 len 36864 29/30 Write block 409 len 4096 data: 14 30/30 Read block 351 len 12288 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 5 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 17 17 15 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:45.259446538Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1876} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 164 164 164 266 266 266 F F F 2 040-059 169 169 169 273 273 273 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 171 171 171 276 276 276 F F F 5 100-119 172 172 172 277 277 277 F F F 6 120-139 169 169 169 273 273 273 F F F 7 140-159 168 168 168 272 272 272 F F F 8 160-179 171 171 171 276 276 276 F F F 9 180-199 171 171 171 276 276 276 F F F 10 200-219 169 169 169 273 273 273 F F F 11 220-239 169 169 169 273 273 273 F F F 12 240-259 172 172 172 277 277 277 F F F 13 260-279 171 171 171 276 276 276 F F F 14 280-299 168 168 168 272 272 272 T T T 15 300-319 156 156 156 255 255 255 T T T 16 320-339 172 172 172 277 277 277 F F F 17 340-359 172 172 172 277 277 277 T T T 18 360-379 170 170 170 275 275 275 F F F 19 380-399 169 169 169 273 273 273 F F F 20 400-419 169 169 169 273 273 273 T T T 21 420-439 169 169 169 273 273 273 F F F 22 440-459 168 168 168 272 272 272 F F F 23 460-479 168 168 168 272 272 272 F F F 24 480-499 163 163 163 264 264 264 F F F 25 500-519 165 165 165 268 268 268 F F F 26 520-539 170 170 170 275 275 275 T T T 27 540-559 172 172 172 277 277 277 T T T 28 560-579 171 171 171 276 276 276 T T T 29 580-599 171 171 171 276 276 276 F F F Max gen: 172, Max flush: 277 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 F F F 1 020-039 164 164 164 266 266 266 F F F 2 040-059 169 169 169 273 273 273 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 171 171 171 276 276 276 F F F 5 100-119 172 172 172 277 277 277 F F F 6 120-139 169 169 169 273 273 273 F F F 7 140-159 168 168 168 272 272 272 F F F 8 160-179 171 171 171 276 276 276 F F F 9 180-199 171 171 171 276 276 276 F F F 10 200-219 169 169 169 273 273 273 F F F 11 220-239 169 169 169 273 273 273 F F F 12 240-259 172 172 172 277 277 277 F F F 13 260-279 171 171 171 276 276 276 F F F 14 280-299 173 173 173 278 278 278 F F F 15 300-319 173 173 173 278 278 278 F F F 16 320-339 172 172 172 277 277 277 F F F 17 340-359 173 173 173 278 278 278 F F F 18 360-379 170 170 170 275 275 275 F F F 19 380-399 169 169 169 273 273 273 F F F 20 400-419 173 173 173 278 278 278 F F F 21 420-439 169 169 169 273 273 273 F F F 22 440-459 168 168 168 272 272 272 F F F 23 460-479 168 168 168 272 272 272 F F F 24 480-499 163 163 163 264 264 264 F F F 25 500-519 165 165 165 268 268 268 F F F 26 520-539 173 173 173 278 278 278 F F F 27 540-559 173 173 173 278 278 278 F F F 28 560-579 173 173 173 278 278 278 F F F 29 580-599 171 171 171 276 276 276 F F F Max gen: 173, Max flush: 278 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.512438167Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.558738828Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.559692313Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.559734612Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.559748966Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.559764079Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884} {"msg":"Crucible ee3bb1f2-880c-421d-824e-f7db606f0885 has session id: 519cfd14-c4bc-4c79-ad12-b5e8f05df900","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.560382082Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Upstairs opts: Upstairs UUID: ee3bb1f2-880c-421d-824e-f7db606f0885, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.560444565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Crucible stats registered with UUID: ee3bb1f2-880c-421d-824e-f7db606f0885","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.560478963Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:49.560501858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.560966158Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.560995284Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884} {"msg":"ee3bb1f2-880c-421d-824e-f7db606f0885 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.561044055Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.561063099Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"0","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.561095266Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"0","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.56111296Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"1","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.561130924Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"1","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.561184935Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"2","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.561225104Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"2","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.561248789Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.561264242Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.561322005Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"ds_connection connected from Ok(127.0.0.1:39155)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.561360373Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"ds_connection connected from Ok(127.0.0.1:59349)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.561387929Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:49.561589923Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:49.561634492Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:49.561661318Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.561719691Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.562145823Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"0","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.562186041Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"2","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.562775128Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"2","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.562928692Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"0","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.5633857Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:49.563531672Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.562938756Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"ds_connection connected from Ok(127.0.0.1:34813)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.563040267Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.564097755Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"1","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.564645424Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"1","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"[0]R flush_numbers[0..12]: [254, 266, 273, 268, 276, 277, 273, 272, 276, 276, 273, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565340874Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"[0]R generation[0..12]: [156, 164, 169, 165, 171, 172, 169, 168, 171, 171, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565361788Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565376411Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"[1]R flush_numbers[0..12]: [254, 266, 273, 268, 276, 277, 273, 272, 276, 276, 273, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565389894Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"[1]R generation[0..12]: [156, 164, 169, 165, 171, 172, 169, 168, 171, 171, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565403997Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.56541765Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"[2]R flush_numbers[0..12]: [254, 266, 273, 268, 276, 277, 273, 272, 276, 276, 273, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565431153Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"[2]R generation[0..12]: [156, 164, 169, 165, 171, 172, 169, 168, 171, 171, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565444446Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565458839Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565471202Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565483915Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565496347Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.56550878Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"":"downstairs","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565521483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:31:59.565535976Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"0","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.56555407Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"1","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565566812Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"client":"2","":"downstairs","session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565579635Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565591528Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"ee3bb1f2-880c-421d-824e-f7db606f0885 is now active with session: 519cfd14-c4bc-4c79-ad12-b5e8f05df900","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565603771Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565615403Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884,"session_id":"519cfd14-c4bc-4c79-ad12-b5e8f05df900"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565627205Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884} {"msg":"Activated sub_volume ee3bb1f2-880c-421d-824e-f7db606f0885","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:31:59.565638038Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 369 len 16384 data: 8 10 10 8 02/30 Read block 294 len 12288 03/30 Read block 532 len 28672 04/30 Write block 252 len 32768 data: 14 13 15 13 12 13 15 14 05/30 Read block 434 len 24576 06/30 Write block 585 len 4096 data: 16 07/30 Write block 287 len 20480 data: 17 18 17 18 14 08/30 Read block 335 len 32768 09/30 Read block 238 len 40960 10/30 Read block 526 len 32768 11/30 Write block 308 len 40960 data: 15 14 10 12 10 13 12 12 14 14 12/30 Flush 13/30 Write block 354 len 32768 data: 7 10 11 13 17 17 18 12 14/30 Read block 348 len 12288 15/30 Read block 480 len 32768 16/30 Write block 80 len 36864 data: 10 8 7 9 11 12 12 11 11 17/30 Read block 388 len 28672 18/30 Read block 421 len 16384 19/30 Write block 97 len 12288 data: 10 10 11 20/30 Flush 21/30 Flush 22/30 Write block 590 len 36864 data: 15 16 14 16 13 11 12 10 7 23/30 Read block 483 len 32768 24/30 Write block 215 len 16384 data: 8 10 11 10 25/30 Write block 164 len 28672 data: 18 21 19 13 14 17 14 26/30 Read block 63 len 4096 27/30 Read block 523 len 8192 28/30 Read block 570 len 16384 29/30 Write block 19 len 28672 data: 10 11 9 11 12 10 6 30/30 Read block 372 len 16384 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:31:59.588480967Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1884} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 254 254 254 T T T 1 020-039 164 164 164 266 266 266 T T T 2 040-059 169 169 169 273 273 273 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 174 174 174 280 280 280 F F F 5 100-119 172 172 172 277 277 277 F F F 6 120-139 169 169 169 273 273 273 F F F 7 140-159 168 168 168 272 272 272 F F F 8 160-179 171 171 171 276 276 276 T T T 9 180-199 171 171 171 276 276 276 F F F 10 200-219 169 169 169 273 273 273 T T T 11 220-239 169 169 169 273 273 273 F F F 12 240-259 174 174 174 279 279 279 F F F 13 260-279 171 171 171 276 276 276 F F F 14 280-299 174 174 174 279 279 279 F F F 15 300-319 174 174 174 279 279 279 F F F 16 320-339 172 172 172 277 277 277 F F F 17 340-359 174 174 174 280 280 280 F F F 18 360-379 174 174 174 280 280 280 F F F 19 380-399 169 169 169 273 273 273 F F F 20 400-419 173 173 173 278 278 278 F F F 21 420-439 169 169 169 273 273 273 F F F 22 440-459 168 168 168 272 272 272 F F F 23 460-479 168 168 168 272 272 272 F F F 24 480-499 163 163 163 264 264 264 F F F 25 500-519 165 165 165 268 268 268 F F F 26 520-539 173 173 173 278 278 278 F F F 27 540-559 173 173 173 278 278 278 F F F 28 560-579 173 173 173 278 278 278 F F F 29 580-599 174 174 174 279 279 279 T T T Max gen: 174, Max flush: 280 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 281 281 281 F F F 1 020-039 175 175 175 281 281 281 F F F 2 040-059 169 169 169 273 273 273 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 174 174 174 280 280 280 F F F 5 100-119 172 172 172 277 277 277 F F F 6 120-139 169 169 169 273 273 273 F F F 7 140-159 168 168 168 272 272 272 F F F 8 160-179 175 175 175 281 281 281 F F F 9 180-199 171 171 171 276 276 276 F F F 10 200-219 175 175 175 281 281 281 F F F 11 220-239 169 169 169 273 273 273 F F F 12 240-259 174 174 174 279 279 279 F F F 13 260-279 171 171 171 276 276 276 F F F 14 280-299 174 174 174 279 279 279 F F F 15 300-319 174 174 174 279 279 279 F F F 16 320-339 172 172 172 277 277 277 F F F 17 340-359 174 174 174 280 280 280 F F F 18 360-379 174 174 174 280 280 280 F F F 19 380-399 169 169 169 273 273 273 F F F 20 400-419 173 173 173 278 278 278 F F F 21 420-439 169 169 169 273 273 273 F F F 22 440-459 168 168 168 272 272 272 F F F 23 460-479 168 168 168 272 272 272 F F F 24 480-499 163 163 163 264 264 264 F F F 25 500-519 165 165 165 268 268 268 F F F 26 520-539 173 173 173 278 278 278 F F F 27 540-559 173 173 173 278 278 278 F F F 28 560-579 173 173 173 278 278 278 F F F 29 580-599 175 175 175 281 281 281 F F F Max gen: 175, Max flush: 281 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.833221544Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.877897194Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.878910223Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.878955552Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.878970476Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.878985149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892} {"msg":"Crucible 505ac626-b9a1-4a3b-ac37-b57014b93308 has session id: dc96d98e-2912-4099-90a2-81f5d9444491","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.879700563Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Upstairs opts: Upstairs UUID: 505ac626-b9a1-4a3b-ac37-b57014b93308, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.879745033Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Crucible stats registered with UUID: 505ac626-b9a1-4a3b-ac37-b57014b93308","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.879768538Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:03.879789342Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880163433Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.8801969Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892} {"msg":"505ac626-b9a1-4a3b-ac37-b57014b93308 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880232848Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880251322Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"0","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880279858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"0","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880293701Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"1","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880308034Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"1","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880321217Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"2","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880342432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"2","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880367437Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880391962Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880417738Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:03.880447434Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:03.880477291Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:03.880508447Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"ds_connection connected from Ok(127.0.0.1:59130)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880533003Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"ds_connection connected from Ok(127.0.0.1:45598)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.880559549Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.88061153Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.881280564Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"1","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.881467154Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"0","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.881855128Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"1","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.881991878Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"0","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.882424411Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:03.882539156Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.882452808Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"ds_connection connected from Ok(127.0.0.1:42658)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.882534395Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.883662559Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"2","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.884298216Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"2","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"[0]R flush_numbers[0..12]: [281, 281, 273, 268, 280, 277, 273, 272, 281, 276, 281, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.884884342Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"[0]R generation[0..12]: [175, 175, 169, 165, 174, 172, 169, 168, 175, 171, 175, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.884912108Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.884931682Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"[1]R flush_numbers[0..12]: [281, 281, 273, 268, 280, 277, 273, 272, 281, 276, 281, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.884954607Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"[1]R generation[0..12]: [175, 175, 169, 165, 174, 172, 169, 168, 175, 171, 175, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.884973051Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.884990495Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"[2]R flush_numbers[0..12]: [281, 281, 273, 268, 280, 277, 273, 272, 281, 276, 281, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885007759Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"[2]R generation[0..12]: [175, 175, 169, 165, 174, 172, 169, 168, 175, 171, 175, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885024613Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885041406Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885057129Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885073173Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885088496Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.88510422Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"":"downstairs","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885120353Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885134876Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"0","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885160692Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"1","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885177015Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"client":"2","":"downstairs","session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.88519712Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885212943Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"505ac626-b9a1-4a3b-ac37-b57014b93308 is now active with session: dc96d98e-2912-4099-90a2-81f5d9444491","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885228807Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.88524445Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892,"session_id":"dc96d98e-2912-4099-90a2-81f5d9444491"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885259303Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892} {"msg":"Activated sub_volume 505ac626-b9a1-4a3b-ac37-b57014b93308","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:13.885272936Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 86 len 4096 data: 13 02/30 Read block 476 len 20480 03/30 Write block 18 len 16384 data: 9 11 12 10 04/30 Write block 17 len 16384 data: 8 10 12 13 05/30 Read block 556 len 16384 06/30 Read block 518 len 4096 07/30 Write block 548 len 8192 data: 15 16 08/30 Read block 33 len 40960 09/30 Read block 379 len 32768 10/30 Read block 289 len 36864 11/30 Read block 500 len 40960 12/30 Write block 386 len 20480 data: 14 12 14 14 17 13/30 Write block 159 len 32768 data: 14 14 13 10 15 19 22 20 14/30 Write block 319 len 24576 data: 13 14 14 13 13 13 15/30 Read block 321 len 20480 16/30 Read block 126 len 36864 17/30 Read block 265 len 32768 18/30 Write block 474 len 36864 data: 13 14 10 10 9 9 11 10 12 19/30 Read block 146 len 16384 20/30 Read block 343 len 12288 21/30 Write block 561 len 16384 data: 20 20 20 19 22/30 Read block 16 len 28672 23/30 Read block 242 len 12288 24/30 Write block 86 len 4096 data: 14 25/30 Read block 3 len 16384 26/30 Read block 159 len 12288 27/30 Write block 183 len 24576 data: 19 23 19 15 14 13 28/30 Write block 165 len 8192 data: 23 21 29/30 Write block 577 len 28672 data: 14 15 17 13 14 14 14 30/30 Write block 426 len 8192 data: 12 14 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:13.904744988Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1892} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 281 281 281 T T T 1 020-039 175 175 175 281 281 281 T T T 2 040-059 169 169 169 273 273 273 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 174 174 174 280 280 280 T T T 5 100-119 172 172 172 277 277 277 F F F 6 120-139 169 169 169 273 273 273 F F F 7 140-159 168 168 168 272 272 272 T T T 8 160-179 175 175 175 281 281 281 T T T 9 180-199 171 171 171 276 276 276 T T T 10 200-219 175 175 175 281 281 281 F F F 11 220-239 169 169 169 273 273 273 F F F 12 240-259 174 174 174 279 279 279 F F F 13 260-279 171 171 171 276 276 276 F F F 14 280-299 174 174 174 279 279 279 F F F 15 300-319 174 174 174 279 279 279 T T T 16 320-339 172 172 172 277 277 277 T T T 17 340-359 174 174 174 280 280 280 F F F 18 360-379 174 174 174 280 280 280 F F F 19 380-399 169 169 169 273 273 273 T T T 20 400-419 173 173 173 278 278 278 F F F 21 420-439 169 169 169 273 273 273 T T T 22 440-459 168 168 168 272 272 272 F F F 23 460-479 168 168 168 272 272 272 T T T 24 480-499 163 163 163 264 264 264 T T T 25 500-519 165 165 165 268 268 268 F F F 26 520-539 173 173 173 278 278 278 F F F 27 540-559 173 173 173 278 278 278 T T T 28 560-579 173 173 173 278 278 278 T T T 29 580-599 175 175 175 281 281 281 T T T Max gen: 175, Max flush: 281 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 282 282 282 F F F 1 020-039 176 176 176 282 282 282 F F F 2 040-059 169 169 169 273 273 273 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 176 176 176 282 282 282 F F F 5 100-119 172 172 172 277 277 277 F F F 6 120-139 169 169 169 273 273 273 F F F 7 140-159 176 176 176 282 282 282 F F F 8 160-179 176 176 176 282 282 282 F F F 9 180-199 176 176 176 282 282 282 F F F 10 200-219 175 175 175 281 281 281 F F F 11 220-239 169 169 169 273 273 273 F F F 12 240-259 174 174 174 279 279 279 F F F 13 260-279 171 171 171 276 276 276 F F F 14 280-299 174 174 174 279 279 279 F F F 15 300-319 176 176 176 282 282 282 F F F 16 320-339 176 176 176 282 282 282 F F F 17 340-359 174 174 174 280 280 280 F F F 18 360-379 174 174 174 280 280 280 F F F 19 380-399 176 176 176 282 282 282 F F F 20 400-419 173 173 173 278 278 278 F F F 21 420-439 176 176 176 282 282 282 F F F 22 440-459 168 168 168 272 272 272 F F F 23 460-479 176 176 176 282 282 282 F F F 24 480-499 176 176 176 282 282 282 F F F 25 500-519 165 165 165 268 268 268 F F F 26 520-539 173 173 173 278 278 278 F F F 27 540-559 176 176 176 282 282 282 F F F 28 560-579 176 176 176 282 282 282 F F F 29 580-599 176 176 176 282 282 282 F F F Max gen: 176, Max flush: 282 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.199861368Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:28.242169066Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.242552739Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.273689958Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.274289707Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.274324164Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.274337937Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.27435204Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900} {"msg":"Crucible ec8f2747-b964-4421-a973-8ade493385c1 has session id: e3347997-dfe5-472d-9a6d-f5f3dc93b5e1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.274975545Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Upstairs opts: Upstairs UUID: ec8f2747-b964-4421-a973-8ade493385c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275010712Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Crucible stats registered with UUID: ec8f2747-b964-4421-a973-8ade493385c1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275025896Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:28.275041279Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275488826Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.27551069Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900} {"msg":"ec8f2747-b964-4421-a973-8ade493385c1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275561871Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275587377Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"0","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275619424Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"0","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275643989Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"1","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275670855Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"1","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275697621Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"2","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275727797Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"2","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275755233Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275781789Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275810405Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:49747)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275837641Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42065)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.275864187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37162)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.2759719Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.2765284Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"1","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.276556776Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"2","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.276572649Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"0","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.277168008Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"1","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.277196914Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"2","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.277219059Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"0","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.277816168Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.277851565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"[0]R flush_numbers[0..12]: [282, 282, 273, 268, 282, 277, 273, 282, 282, 282, 281, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.277909038Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"[0]R generation[0..12]: [176, 176, 169, 165, 176, 172, 169, 176, 176, 176, 175, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.277938404Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.277953987Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"[1]R flush_numbers[0..12]: [282, 282, 273, 268, 282, 277, 273, 282, 282, 282, 281, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.277967941Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"[1]R generation[0..12]: [176, 176, 169, 165, 176, 172, 169, 176, 176, 176, 175, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.277981513Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.277999187Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"[2]R flush_numbers[0..12]: [282, 282, 273, 268, 282, 277, 273, 282, 282, 282, 281, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.27801332Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"[2]R generation[0..12]: [176, 176, 169, 165, 176, 172, 169, 176, 176, 176, 175, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278026943Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278040726Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Max found gen is 177","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278053419Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Generation requested: 178 >= found:177","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-17T22:32:28.278066342Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278086726Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278099099Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"":"downstairs","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278112002Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278123694Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"0","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278136637Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"1","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.27814948Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"client":"2","":"downstairs","session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278162432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278174355Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"ec8f2747-b964-4421-a973-8ade493385c1 is now active with session: e3347997-dfe5-472d-9a6d-f5f3dc93b5e1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278186668Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278201501Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900,"session_id":"e3347997-dfe5-472d-9a6d-f5f3dc93b5e1"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278213894Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900} {"msg":"Activated sub_volume ec8f2747-b964-4421-a973-8ade493385c1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:28.278224846Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 7 len 8192 data: 15 13 02/30 Write block 145 len 36864 data: 13 12 13 15 17 16 20 19 19 03/30 Flush 04/30 Write block 446 len 32768 data: 14 12 10 10 8 9 11 14 05/30 Read block 114 len 24576 06/30 Flush 07/30 Write block 391 len 32768 data: 12 12 16 15 13 12 12 12 08/30 Write block 307 len 8192 data: 14 16 09/30 Read block 385 len 32768 10/30 Read block 243 len 28672 11/30 Write block 319 len 32768 data: 14 15 15 14 14 14 15 14 12/30 Read block 177 len 12288 13/30 Read block 458 len 20480 14/30 Write block 405 len 32768 data: 15 16 18 17 15 15 15 13 15/30 Read block 565 len 8192 16/30 Read block 40 len 12288 17/30 Read block 515 len 12288 18/30 Write block 574 len 20480 data: 11 12 12 15 16 19/30 Read block 79 len 32768 20/30 Read block 583 len 8192 21/30 Read block 588 len 40960 22/30 Read block 383 len 12288 23/30 Read block 399 len 24576 24/30 Read block 175 len 28672 25/30 Read block 269 len 40960 26/30 Write block 351 len 40960 data: 10 9 10 8 11 12 14 18 18 19 27/30 Write block 527 len 32768 data: 17 14 15 12 15 13 13 15 28/30 Write block 504 len 12288 data: 14 13 11 29/30 Read block 36 len 32768 30/30 Write block 527 len 24576 data: 18 15 16 13 16 14 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:28.299683176Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1900} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 176 176 176 282 282 282 F F F 2 040-059 169 169 169 273 273 273 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 176 176 176 282 282 282 F F F 5 100-119 172 172 172 277 277 277 F F F 6 120-139 169 169 169 273 273 273 F F F 7 140-159 178 178 178 283 283 283 F F F 8 160-179 176 176 176 282 282 282 F F F 9 180-199 176 176 176 282 282 282 F F F 10 200-219 175 175 175 281 281 281 F F F 11 220-239 169 169 169 273 273 273 F F F 12 240-259 174 174 174 279 279 279 F F F 13 260-279 171 171 171 276 276 276 F F F 14 280-299 174 174 174 279 279 279 F F F 15 300-319 176 176 176 282 282 282 T T T 16 320-339 176 176 176 282 282 282 T T T 17 340-359 174 174 174 280 280 280 T T T 18 360-379 174 174 174 280 280 280 T T T 19 380-399 176 176 176 282 282 282 T T T 20 400-419 173 173 173 278 278 278 T T T 21 420-439 176 176 176 282 282 282 F F F 22 440-459 178 178 178 284 284 284 F F F 23 460-479 176 176 176 282 282 282 F F F 24 480-499 176 176 176 282 282 282 F F F 25 500-519 165 165 165 268 268 268 T T T 26 520-539 173 173 173 278 278 278 T T T 27 540-559 176 176 176 282 282 282 F F F 28 560-579 176 176 176 282 282 282 T T T 29 580-599 176 176 176 282 282 282 F F F Max gen: 178, Max flush: 284 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 176 176 176 282 282 282 F F F 2 040-059 169 169 169 273 273 273 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 176 176 176 282 282 282 F F F 5 100-119 172 172 172 277 277 277 F F F 6 120-139 169 169 169 273 273 273 F F F 7 140-159 178 178 178 283 283 283 F F F 8 160-179 176 176 176 282 282 282 F F F 9 180-199 176 176 176 282 282 282 F F F 10 200-219 175 175 175 281 281 281 F F F 11 220-239 169 169 169 273 273 273 F F F 12 240-259 174 174 174 279 279 279 F F F 13 260-279 171 171 171 276 276 276 F F F 14 280-299 174 174 174 279 279 279 F F F 15 300-319 179 179 179 285 285 285 F F F 16 320-339 179 179 179 285 285 285 F F F 17 340-359 179 179 179 285 285 285 F F F 18 360-379 179 179 179 285 285 285 F F F 19 380-399 179 179 179 285 285 285 F F F 20 400-419 179 179 179 285 285 285 F F F 21 420-439 176 176 176 282 282 282 F F F 22 440-459 178 178 178 284 284 284 F F F 23 460-479 176 176 176 282 282 282 F F F 24 480-499 176 176 176 282 282 282 F F F 25 500-519 179 179 179 285 285 285 F F F 26 520-539 179 179 179 285 285 285 F F F 27 540-559 176 176 176 282 282 282 F F F 28 560-579 179 179 179 285 285 285 F F F 29 580-599 176 176 176 282 282 282 F F F Max gen: 179, Max flush: 285 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.593291368Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.637400016Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.63806812Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.638111699Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.638127252Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.638143006Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908} {"msg":"Crucible 65e36b2a-21c4-454f-b18b-ef379c5600f2 has session id: feb2912f-7f95-48f9-8985-1efeeeaba354","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.638705377Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Upstairs opts: Upstairs UUID: 65e36b2a-21c4-454f-b18b-ef379c5600f2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.638781323Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Crucible stats registered with UUID: 65e36b2a-21c4-454f-b18b-ef379c5600f2","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.638815581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:32.638831454Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.639234601Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.639268529Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908} {"msg":"65e36b2a-21c4-454f-b18b-ef379c5600f2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.639325161Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.639351297Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"0","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.639379643Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"0","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.639398087Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"1","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.63941353Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"1","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.63946217Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"2","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.639533586Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"2","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.639567033Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.639593609Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.639620055Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"ds_connection connected from Ok(127.0.0.1:47999)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.6396453Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:32.639676687Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:32.639713255Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"ds_connection connected from Ok(127.0.0.1:59104)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.639746392Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:32.639779299Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.639897835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.640415666Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"2","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.640461176Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"0","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.641131671Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"0","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.641160447Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"2","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.641726469Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:32.641848916Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.641529408Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"ds_connection connected from Ok(127.0.0.1:34697)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.641649774Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.642679046Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"1","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.643251389Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"1","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"[0]R flush_numbers[0..12]: [283, 282, 273, 268, 282, 277, 273, 283, 282, 282, 281, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.643842457Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"[0]R generation[0..12]: [178, 176, 169, 165, 176, 172, 169, 178, 176, 176, 175, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.643866372Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.643881745Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"[1]R flush_numbers[0..12]: [283, 282, 273, 268, 282, 277, 273, 283, 282, 282, 281, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.643895578Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"[1]R generation[0..12]: [178, 176, 169, 165, 176, 172, 169, 178, 176, 176, 175, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.643909111Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.643927545Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"[2]R flush_numbers[0..12]: [283, 282, 273, 268, 282, 277, 273, 283, 282, 282, 281, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.643941798Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"[2]R generation[0..12]: [178, 176, 169, 165, 176, 172, 169, 178, 176, 176, 175, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.643955471Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.643969644Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.643987088Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.644010703Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.644034378Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:32:42.644057883Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"":"downstairs","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.64408848Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.644112375Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"0","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.644130149Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"1","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.644143212Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"client":"2","":"downstairs","session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.644158735Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.644180129Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"65e36b2a-21c4-454f-b18b-ef379c5600f2 is now active with session: feb2912f-7f95-48f9-8985-1efeeeaba354","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.644204685Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.644225199Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.644247034Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908} {"msg":"Activated sub_volume 65e36b2a-21c4-454f-b18b-ef379c5600f2","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:42.644267428Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 347 len 8192 data: 7 8 02/30 Write block 574 len 24576 data: 12 13 13 16 17 18 03/30 Read block 128 len 36864 04/30 Flush 05/30 Write block 422 len 24576 data: 8 10 11 9 13 15 06/30 Write block 50 len 32768 data: 16 13 14 12 12 14 12 12 07/30 Write block 138 len 32768 data: 14 14 13 12 10 12 12 14 08/30 Write block 296 len 4096 data: 13 09/30 Read block 173 len 32768 10/30 Read block 427 len 36864 11/30 Read block 356 len 4096 12/30 Write block 86 len 24576 data: 15 12 12 9 12 12 13/30 Flush 14/30 Write block 145 len 24576 data: 15 13 14 16 18 17 15/30 Read block 291 len 20480 16/30 Read block 161 len 12288 17/30 Write block 291 len 32768 data: 15 14 11 11 9 14 15 14 18/30 Write block 295 len 4096 data: 10 19/30 Read block 587 len 12288 20/30 Read block 467 len 28672 21/30 Write block 431 len 24576 data: 15 14 14 13 13 11 22/30 Read block 466 len 12288 23/30 Write block 537 len 40960 data: 15 14 15 15 15 17 19 19 16 18 24/30 Read block 76 len 28672 25/30 Read block 372 len 28672 26/30 Read block 379 len 8192 27/30 Write block 279 len 32768 data: 10 14 14 14 15 17 19 18 28/30 Write block 90 len 20480 data: 13 13 11 10 7 29/30 Write block 377 len 16384 data: 11 12 15 13 30/30 Write block 539 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 6 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 8 Sent Sent Sent false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 4 12 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.664232566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908} Wrote out file "/tmp/test_repair_verify.data" {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.66513109Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.66517561Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665202776Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.66522384Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665246615Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.6652706Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665294836Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665319951Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665342446Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665365981Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665388616Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665402709Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665418942Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665440147Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665461901Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665482616Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665495609Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665507561Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665522254Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665534437Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.66554619Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665557972Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665569665Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665581397Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.66559313Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665604742Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665616685Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665628688Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.66564062Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665652362Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665664035Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665676158Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.66568794Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665699633Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665711325Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665724888Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665736861Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665748573Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665760336Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665772028Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665783641Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665795324Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665806926Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665818658Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665830371Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665842043Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665853766Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665865559Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665877231Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665888854Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665900546Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665912319Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665926002Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665937784Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665949407Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665961069Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665972762Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665984394Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.665996087Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666007639Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666019442Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666031124Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666042757Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666054469Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666066172Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666077965Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666089967Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.66610175Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666113572Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666127505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666139468Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.66615113Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666162823Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666174805Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666186708Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.66619858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666210513Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666222196Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666233988Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666245711Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666257363Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666268996Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666280608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666292301Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666303933Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666315546Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666328979Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666340861Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666352554Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666364336Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666375919Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666387671Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666399134Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666410796Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666422349Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666434001Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666445694Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666457306Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666469019Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666480741Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666492454Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666504176Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666515849Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666529322Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666540984Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666552577Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666564099Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666575712Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666587314Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666598917Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666610649Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666622312Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666633924Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666645557Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666657419Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666669132Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666680755Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666692387Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666704069Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666715732Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666728885Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:42.666740628Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1908,"session_id":"feb2912f-7f95-48f9-8985-1efeeeaba354"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 176 176 176 282 282 282 F F F 2 040-059 180 180 180 287 287 287 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 180 180 180 287 287 287 T T T 5 100-119 172 172 172 277 277 277 F F F 6 120-139 180 180 180 287 287 287 F F F 7 140-159 180 180 180 287 287 287 T T T 8 160-179 176 176 176 282 282 282 F F F 9 180-199 176 176 176 282 282 282 F F F 10 200-219 175 175 175 281 281 281 F F F 11 220-239 169 169 169 273 273 273 F F F 12 240-259 174 174 174 279 279 279 F F F 13 260-279 171 171 171 276 276 276 T T T 14 280-299 180 180 180 287 287 287 T T T 15 300-319 179 179 179 285 285 285 F F F 16 320-339 179 179 179 285 285 285 F F F 17 340-359 180 180 180 286 286 286 F F F 18 360-379 179 179 179 285 285 285 T T T 19 380-399 179 179 179 285 285 285 T T T 20 400-419 179 179 179 285 285 285 F F F 21 420-439 180 180 180 287 287 287 T T T 22 440-459 178 178 178 284 284 284 F F F 23 460-479 176 176 176 282 282 282 F F F 24 480-499 176 176 176 282 282 282 F F F 25 500-519 179 179 179 285 285 285 F F F 26 520-539 179 179 179 285 285 285 T T T 27 540-559 176 176 176 282 282 282 T T T 28 560-579 180 180 180 286 286 286 F F F 29 580-599 176 176 176 282 282 282 F F F Max gen: 180, Max flush: 287 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 283 283 283 F F F 1 020-039 176 176 176 282 282 282 F F F 2 040-059 180 180 180 287 287 287 F F F 3 060-079 165 165 165 268 268 268 F F F 4 080-099 181 181 181 288 288 288 F F F 5 100-119 172 172 172 277 277 277 F F F 6 120-139 180 180 180 287 287 287 F F F 7 140-159 181 181 181 288 288 288 F F F 8 160-179 176 176 176 282 282 282 F F F 9 180-199 176 176 176 282 282 282 F F F 10 200-219 175 175 175 281 281 281 F F F 11 220-239 169 169 169 273 273 273 F F F 12 240-259 174 174 174 279 279 279 F F F 13 260-279 181 181 181 288 288 288 F F F 14 280-299 181 181 181 288 288 288 F F F 15 300-319 179 179 179 285 285 285 F F F 16 320-339 179 179 179 285 285 285 F F F 17 340-359 180 180 180 286 286 286 F F F 18 360-379 181 181 181 288 288 288 F F F 19 380-399 181 181 181 288 288 288 F F F 20 400-419 179 179 179 285 285 285 F F F 21 420-439 181 181 181 288 288 288 F F F 22 440-459 178 178 178 284 284 284 F F F 23 460-479 176 176 176 282 282 282 F F F 24 480-499 176 176 176 282 282 282 F F F 25 500-519 179 179 179 285 285 285 F F F 26 520-539 181 181 181 288 288 288 F F F 27 540-559 181 181 181 288 288 288 F F F 28 560-579 180 180 180 286 286 286 F F F 29 580-599 176 176 176 282 282 282 F F F Max gen: 181, Max flush: 288 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:46.927022039Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:46.971986701Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:46.972360231Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.003802116Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.004434082Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.00447289Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.004489454Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.004506938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916} {"msg":"Crucible 2e0ccb6d-67a9-4c29-8a44-82c0e724af85 has session id: d58644d7-2412-405c-8a10-d3183ceda31c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005051975Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Upstairs opts: Upstairs UUID: 2e0ccb6d-67a9-4c29-8a44-82c0e724af85, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005103817Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Crucible stats registered with UUID: 2e0ccb6d-67a9-4c29-8a44-82c0e724af85","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005134843Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:47.005160269Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005482778Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005508664Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916} {"msg":"2e0ccb6d-67a9-4c29-8a44-82c0e724af85 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005554534Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005571407Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"0","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005598663Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"0","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005612466Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"1","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005626949Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"1","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005640462Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"2","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005664478Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"2","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005689553Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005715108Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005742274Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38982)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.00577009Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"ds_connection connected from Ok(127.0.0.1:45251)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.005796746Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57717)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.006007241Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.006458069Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"2","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.006628385Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"1","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.006670644Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"0","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.007201759Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"2","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.007230805Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"1","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.007329887Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"0","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.007803949Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.007931267Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"[0]R flush_numbers[0..12]: [283, 282, 287, 268, 288, 277, 287, 288, 282, 282, 281, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.007999911Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"[0]R generation[0..12]: [178, 176, 180, 165, 181, 172, 180, 181, 176, 176, 175, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008026107Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.00804109Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"[1]R flush_numbers[0..12]: [283, 282, 287, 268, 288, 277, 287, 288, 282, 282, 281, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008055093Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"[1]R generation[0..12]: [178, 176, 180, 165, 181, 172, 180, 181, 176, 176, 175, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008068676Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008082479Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"[2]R flush_numbers[0..12]: [283, 282, 287, 268, 288, 277, 287, 288, 282, 282, 281, 273]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008095892Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"[2]R generation[0..12]: [178, 176, 180, 165, 181, 172, 180, 181, 176, 176, 175, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008109255Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008127709Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008141182Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008154175Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008167627Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.00818031Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008200445Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008212507Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"0","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.00822557Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"1","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008238403Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"client":"2","":"downstairs","session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008251626Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008263658Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"2e0ccb6d-67a9-4c29-8a44-82c0e724af85 is now active with session: d58644d7-2412-405c-8a10-d3183ceda31c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008276141Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008287853Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916,"session_id":"d58644d7-2412-405c-8a10-d3183ceda31c"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008299866Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916} {"msg":"Activated sub_volume 2e0ccb6d-67a9-4c29-8a44-82c0e724af85","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:47.008310918Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 584 len 32768 data: 16 17 13 14 12 15 16 17 02/30 Read block 577 len 16384 03/30 Read block 419 len 12288 04/30 Read block 107 len 20480 05/30 Read block 257 len 32768 06/30 Write block 6 len 4096 data: 14 07/30 Write block 211 len 24576 data: 17 14 9 7 9 11 08/30 Flush 09/30 Write block 244 len 40960 data: 9 10 14 15 13 13 13 14 15 14 10/30 Write block 557 len 40960 data: 15 17 18 20 21 21 21 20 18 15 11/30 Read block 217 len 24576 12/30 Write block 225 len 4096 data: 11 13/30 Write block 335 len 4096 data: 22 14/30 Read block 235 len 20480 15/30 Read block 572 len 40960 16/30 Write block 68 len 16384 data: 12 15 16 16 17/30 Read block 236 len 20480 18/30 Write block 499 len 28672 data: 13 13 18 17 17 15 14 19/30 Write block 429 len 36864 data: 14 16 16 15 15 14 14 12 9 20/30 Read block 70 len 32768 21/30 Flush 22/30 Write block 197 len 12288 data: 14 14 12 23/30 Read block 381 len 36864 24/30 Read block 100 len 4096 25/30 Read block 23 len 4096 26/30 Read block 71 len 28672 27/30 Write block 444 len 40960 data: 13 14 15 13 11 11 9 10 12 15 28/30 Write block 126 len 24576 data: 7 8 10 12 11 14 29/30 Write block 96 len 36864 data: 7 11 11 12 9 9 10 9 10 30/30 Write block 210 len 12288 data: 16 18 15 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 3 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:47.028254572Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1916} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 289 289 289 F F F 1 020-039 176 176 176 282 282 282 F F F 2 040-059 180 180 180 287 287 287 F F F 3 060-079 182 182 182 290 290 290 F F F 4 080-099 181 181 181 288 288 288 T T T 5 100-119 172 172 172 277 277 277 T T T 6 120-139 180 180 180 287 287 287 T T T 7 140-159 181 181 181 288 288 288 F F F 8 160-179 176 176 176 282 282 282 F F F 9 180-199 176 176 176 282 282 282 T T T 10 200-219 182 182 182 289 289 289 T T T 11 220-239 182 182 182 290 290 290 F F F 12 240-259 182 182 182 290 290 290 F F F 13 260-279 181 181 181 288 288 288 F F F 14 280-299 181 181 181 288 288 288 F F F 15 300-319 179 179 179 285 285 285 F F F 16 320-339 182 182 182 290 290 290 F F F 17 340-359 180 180 180 286 286 286 F F F 18 360-379 181 181 181 288 288 288 F F F 19 380-399 181 181 181 288 288 288 F F F 20 400-419 179 179 179 285 285 285 F F F 21 420-439 182 182 182 290 290 290 F F F 22 440-459 178 178 178 284 284 284 T T T 23 460-479 176 176 176 282 282 282 F F F 24 480-499 182 182 182 290 290 290 F F F 25 500-519 182 182 182 290 290 290 F F F 26 520-539 181 181 181 288 288 288 F F F 27 540-559 182 182 182 290 290 290 F F F 28 560-579 182 182 182 290 290 290 F F F 29 580-599 182 182 182 289 289 289 F F F Max gen: 182, Max flush: 290 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 289 289 289 F F F 1 020-039 176 176 176 282 282 282 F F F 2 040-059 180 180 180 287 287 287 F F F 3 060-079 182 182 182 290 290 290 F F F 4 080-099 183 183 183 291 291 291 F F F 5 100-119 183 183 183 291 291 291 F F F 6 120-139 183 183 183 291 291 291 F F F 7 140-159 181 181 181 288 288 288 F F F 8 160-179 176 176 176 282 282 282 F F F 9 180-199 183 183 183 291 291 291 F F F 10 200-219 183 183 183 291 291 291 F F F 11 220-239 182 182 182 290 290 290 F F F 12 240-259 182 182 182 290 290 290 F F F 13 260-279 181 181 181 288 288 288 F F F 14 280-299 181 181 181 288 288 288 F F F 15 300-319 179 179 179 285 285 285 F F F 16 320-339 182 182 182 290 290 290 F F F 17 340-359 180 180 180 286 286 286 F F F 18 360-379 181 181 181 288 288 288 F F F 19 380-399 181 181 181 288 288 288 F F F 20 400-419 179 179 179 285 285 285 F F F 21 420-439 182 182 182 290 290 290 F F F 22 440-459 183 183 183 291 291 291 F F F 23 460-479 176 176 176 282 282 282 F F F 24 480-499 182 182 182 290 290 290 F F F 25 500-519 182 182 182 290 290 290 F F F 26 520-539 181 181 181 288 288 288 F F F 27 540-559 182 182 182 290 290 290 F F F 28 560-579 182 182 182 290 290 290 F F F 29 580-599 182 182 182 289 289 289 F F F Max gen: 183, Max flush: 291 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.301756326Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.346117628Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.346928553Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.346973063Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.346987866Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.347002169Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924} {"msg":"Crucible 558cc68d-0b99-4d4a-a67a-f2becb301574 has session id: 9bc0b474-fba0-4e09-a9d3-d6cde31e2462","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.347621472Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Upstairs opts: Upstairs UUID: 558cc68d-0b99-4d4a-a67a-f2becb301574, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.347666662Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Crucible stats registered with UUID: 558cc68d-0b99-4d4a-a67a-f2becb301574","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.347691608Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:51.347712642Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348121751Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348154258Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924} {"msg":"558cc68d-0b99-4d4a-a67a-f2becb301574 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348220122Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348244557Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"0","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348273283Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"0","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348287456Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"1","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348302739Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"1","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348324744Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"2","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.34835052Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"2","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348372975Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.34839745Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348422556Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:51.348452182Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:51.348481928Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:32:51.348509264Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"ds_connection connected from Ok(127.0.0.1:62807)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348532029Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348671809Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"ds_connection connected from Ok(127.0.0.1:60857)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.348727211Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.349133459Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"1","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.349329301Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"0","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.349712554Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"1","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.349819877Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"0","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.350294499Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:32:51.350384719Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.35015647Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"ds_connection connected from Ok(127.0.0.1:56302)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.350309083Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.351207157Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"2","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.351767707Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"2","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"[0]R flush_numbers[0..12]: [289, 282, 287, 290, 291, 291, 291, 288, 282, 291, 291, 290]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352452825Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"[0]R generation[0..12]: [182, 176, 180, 182, 183, 183, 183, 181, 176, 183, 183, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352482982Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352502566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"[1]R flush_numbers[0..12]: [289, 282, 287, 290, 291, 291, 291, 288, 282, 291, 291, 290]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.35251977Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"[1]R generation[0..12]: [182, 176, 180, 182, 183, 183, 183, 181, 176, 183, 183, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352536593Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352554047Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"[2]R flush_numbers[0..12]: [289, 282, 287, 290, 291, 291, 291, 288, 282, 291, 291, 290]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352570951Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"[2]R generation[0..12]: [182, 176, 180, 182, 183, 183, 183, 181, 176, 183, 183, 182]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352587354Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352603928Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352619631Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352649108Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Next flush: 292","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352665021Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352680815Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"":"downstairs","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352696808Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352711211Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"0","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352727345Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"1","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352743198Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"client":"2","":"downstairs","session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352759041Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352773984Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"558cc68d-0b99-4d4a-a67a-f2becb301574 is now active with session: 9bc0b474-fba0-4e09-a9d3-d6cde31e2462","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352789508Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352803821Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924,"session_id":"9bc0b474-fba0-4e09-a9d3-d6cde31e2462"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352818594Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924} {"msg":"Activated sub_volume 558cc68d-0b99-4d4a-a67a-f2becb301574","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:01.352832077Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 80 len 32768 data: 11 9 8 10 12 13 16 13 02/30 Read block 23 len 24576 03/30 Read block 557 len 12288 04/30 Read block 175 len 28672 05/30 Flush 06/30 Write block 537 len 16384 data: 16 15 17 16 07/30 Read block 18 len 12288 08/30 Read block 170 len 28672 09/30 Write block 437 len 8192 data: 10 9 10/30 Flush 11/30 Write block 59 len 40960 data: 12 12 13 15 13 15 13 13 14 13 12/30 Read block 529 len 4096 13/30 Read block 541 len 40960 14/30 Flush 15/30 Write block 377 len 12288 data: 12 13 16 16/30 Read block 1 len 4096 17/30 Read block 327 len 24576 18/30 Read block 52 len 20480 19/30 Read block 89 len 20480 20/30 Read block 501 len 4096 21/30 Write block 107 len 36864 data: 9 11 9 9 9 11 13 15 18 22/30 Write block 287 len 24576 data: 18 19 18 19 16 15 23/30 Write block 566 len 24576 data: 16 18 18 18 15 15 24/30 Write block 238 len 4096 data: 12 25/30 Flush 26/30 Write block 99 len 16384 data: 13 10 10 11 27/30 Write block 550 len 28672 data: 14 13 10 11 13 14 16 28/30 Write block 42 len 8192 data: 13 17 29/30 Read block 214 len 4096 30/30 Write block 258 len 12288 data: 16 15 14 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:01.372720108Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1924} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 289 289 289 F F F 1 020-039 176 176 176 282 282 282 F F F 2 040-059 184 184 184 294 294 294 T T T 3 060-079 184 184 184 294 294 294 F F F 4 080-099 184 184 184 292 292 292 T T T 5 100-119 184 184 184 295 295 295 T T T 6 120-139 183 183 183 291 291 291 F F F 7 140-159 181 181 181 288 288 288 F F F 8 160-179 176 176 176 282 282 282 F F F 9 180-199 183 183 183 291 291 291 F F F 10 200-219 183 183 183 291 291 291 F F F 11 220-239 184 184 184 295 295 295 F F F 12 240-259 182 182 182 290 290 290 T T T 13 260-279 181 181 181 288 288 288 T T T 14 280-299 184 184 184 295 295 295 F F F 15 300-319 179 179 179 285 285 285 F F F 16 320-339 182 182 182 290 290 290 F F F 17 340-359 180 180 180 286 286 286 F F F 18 360-379 184 184 184 295 295 295 F F F 19 380-399 181 181 181 288 288 288 F F F 20 400-419 179 179 179 285 285 285 F F F 21 420-439 184 184 184 293 293 293 F F F 22 440-459 183 183 183 291 291 291 F F F 23 460-479 176 176 176 282 282 282 F F F 24 480-499 182 182 182 290 290 290 F F F 25 500-519 182 182 182 290 290 290 F F F 26 520-539 184 184 184 293 293 293 F F F 27 540-559 184 184 184 293 293 293 T T T 28 560-579 184 184 184 295 295 295 F F F 29 580-599 182 182 182 289 289 289 F F F Max gen: 184, Max flush: 295 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 289 289 289 F F F 1 020-039 176 176 176 282 282 282 F F F 2 040-059 185 185 185 296 296 296 F F F 3 060-079 184 184 184 294 294 294 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 185 185 185 296 296 296 F F F 6 120-139 183 183 183 291 291 291 F F F 7 140-159 181 181 181 288 288 288 F F F 8 160-179 176 176 176 282 282 282 F F F 9 180-199 183 183 183 291 291 291 F F F 10 200-219 183 183 183 291 291 291 F F F 11 220-239 184 184 184 295 295 295 F F F 12 240-259 185 185 185 296 296 296 F F F 13 260-279 185 185 185 296 296 296 F F F 14 280-299 184 184 184 295 295 295 F F F 15 300-319 179 179 179 285 285 285 F F F 16 320-339 182 182 182 290 290 290 F F F 17 340-359 180 180 180 286 286 286 F F F 18 360-379 184 184 184 295 295 295 F F F 19 380-399 181 181 181 288 288 288 F F F 20 400-419 179 179 179 285 285 285 F F F 21 420-439 184 184 184 293 293 293 F F F 22 440-459 183 183 183 291 291 291 F F F 23 460-479 176 176 176 282 282 282 F F F 24 480-499 182 182 182 290 290 290 F F F 25 500-519 182 182 182 290 290 290 F F F 26 520-539 184 184 184 293 293 293 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 184 184 184 295 295 295 F F F 29 580-599 182 182 182 289 289 289 F F F Max gen: 185, Max flush: 296 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.621254605Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.665513304Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.66623141Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.666275029Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.666289982Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.666304405Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932} {"msg":"Crucible dae65bed-e63d-4842-920c-7b32fce7ca84 has session id: ec07eb47-d075-4ad5-b266-86cd8c5a4853","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.666958987Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Upstairs opts: Upstairs UUID: dae65bed-e63d-4842-920c-7b32fce7ca84, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.666998285Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Crucible stats registered with UUID: dae65bed-e63d-4842-920c-7b32fce7ca84","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667014628Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:05.667032182Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667432298Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667455483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932} {"msg":"dae65bed-e63d-4842-920c-7b32fce7ca84 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667512606Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.66752965Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"0","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667557205Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"0","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.66757717Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"1","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667603686Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"1","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667627981Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"2","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667655337Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"2","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667679812Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"io task","client":"2","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667704547Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"io task","client":"0","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667730263Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"io task","client":"1","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"ds_connection connected from Ok(127.0.0.1:51971)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667756119Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"io task","client":"2","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"ds_connection connected from Ok(127.0.0.1:48934)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667782724Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"io task","client":"0","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"ds_connection connected from Ok(127.0.0.1:56651)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.667822283Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"io task","client":"1","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.668434505Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"0","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.668464951Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"1","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.669084065Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"0","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.6692002Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"1","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.669673742Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.669700528Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.669714891Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"2","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670242305Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"2","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"[0]R flush_numbers[0..12]: [289, 282, 296, 294, 296, 296, 291, 288, 282, 291, 291, 295]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670771739Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"[0]R generation[0..12]: [182, 176, 185, 184, 185, 185, 183, 181, 176, 183, 183, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670792944Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670807317Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"[1]R flush_numbers[0..12]: [289, 282, 296, 294, 296, 296, 291, 288, 282, 291, 291, 295]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.67082126Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"[1]R generation[0..12]: [182, 176, 185, 184, 185, 185, 183, 181, 176, 183, 183, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670834923Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670853417Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"[2]R flush_numbers[0..12]: [289, 282, 296, 294, 296, 296, 291, 288, 282, 291, 291, 295]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.67086754Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"[2]R generation[0..12]: [182, 176, 185, 184, 185, 185, 183, 181, 176, 183, 183, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670881243Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670894976Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670908489Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670921711Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670934224Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670946967Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"":"downstairs","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:33:05.67095947Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670975953Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"0","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.670989116Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"1","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.671002129Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"client":"2","":"downstairs","session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.671015252Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.671027084Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"dae65bed-e63d-4842-920c-7b32fce7ca84 is now active with session: ec07eb47-d075-4ad5-b266-86cd8c5a4853","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.671039797Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.67105467Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932,"session_id":"ec07eb47-d075-4ad5-b266-86cd8c5a4853"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.671067103Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932} {"msg":"Activated sub_volume dae65bed-e63d-4842-920c-7b32fce7ca84","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:05.671078165Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 401 len 24576 data: 10 12 14 14 16 17 02/30 Read block 286 len 16384 03/30 Read block 164 len 4096 04/30 Read block 324 len 4096 05/30 Read block 78 len 40960 06/30 Write block 444 len 24576 data: 14 15 16 14 12 12 07/30 Write block 525 len 12288 data: 15 17 19 08/30 Write block 170 len 12288 data: 15 13 13 09/30 Read block 369 len 32768 10/30 Write block 57 len 32768 data: 13 14 13 13 14 16 14 16 11/30 Read block 559 len 40960 12/30 Write block 11 len 40960 data: 11 12 10 9 8 9 9 11 13 14 13/30 Write block 365 len 28672 data: 15 10 9 9 9 11 11 14/30 Write block 327 len 36864 data: 14 11 10 11 16 18 19 21 23 15/30 Read block 130 len 24576 16/30 Write block 514 len 20480 data: 11 10 9 15 15 17/30 Write block 279 len 28672 data: 11 15 15 15 16 18 20 18/30 Read block 340 len 16384 19/30 Write block 196 len 40960 data: 13 15 15 13 16 18 15 13 12 14 20/30 Read block 309 len 20480 21/30 Read block 298 len 4096 22/30 Read block 38 len 40960 23/30 Read block 465 len 32768 24/30 Write block 510 len 32768 data: 13 8 10 10 12 11 10 16 25/30 Read block 46 len 20480 26/30 Read block 507 len 32768 27/30 Read block 494 len 28672 28/30 Write block 265 len 36864 data: 17 15 14 9 9 9 9 12 12 29/30 Read block 482 len 32768 30/30 Write block 166 len 28672 data: 22 14 15 18 16 14 14 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:05.693646475Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1932} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 289 289 289 T T T 1 020-039 176 176 176 282 282 282 T T T 2 040-059 185 185 185 296 296 296 T T T 3 060-079 184 184 184 294 294 294 T T T 4 080-099 185 185 185 296 296 296 F F F 5 100-119 185 185 185 296 296 296 F F F 6 120-139 183 183 183 291 291 291 F F F 7 140-159 181 181 181 288 288 288 F F F 8 160-179 176 176 176 282 282 282 T T T 9 180-199 183 183 183 291 291 291 T T T 10 200-219 183 183 183 291 291 291 T T T 11 220-239 184 184 184 295 295 295 F F F 12 240-259 185 185 185 296 296 296 F F F 13 260-279 185 185 185 296 296 296 T T T 14 280-299 184 184 184 295 295 295 T T T 15 300-319 179 179 179 285 285 285 F F F 16 320-339 182 182 182 290 290 290 T T T 17 340-359 180 180 180 286 286 286 F F F 18 360-379 184 184 184 295 295 295 T T T 19 380-399 181 181 181 288 288 288 F F F 20 400-419 179 179 179 285 285 285 T T T 21 420-439 184 184 184 293 293 293 F F F 22 440-459 183 183 183 291 291 291 T T T 23 460-479 176 176 176 282 282 282 F F F 24 480-499 182 182 182 290 290 290 F F F 25 500-519 182 182 182 290 290 290 T T T 26 520-539 184 184 184 293 293 293 T T T 27 540-559 185 185 185 296 296 296 F F F 28 560-579 184 184 184 295 295 295 F F F 29 580-599 182 182 182 289 289 289 F F F Max gen: 185, Max flush: 296 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 297 297 297 F F F 1 020-039 186 186 186 297 297 297 F F F 2 040-059 186 186 186 297 297 297 F F F 3 060-079 186 186 186 297 297 297 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 185 185 185 296 296 296 F F F 6 120-139 183 183 183 291 291 291 F F F 7 140-159 181 181 181 288 288 288 F F F 8 160-179 186 186 186 297 297 297 F F F 9 180-199 186 186 186 297 297 297 F F F 10 200-219 186 186 186 297 297 297 F F F 11 220-239 184 184 184 295 295 295 F F F 12 240-259 185 185 185 296 296 296 F F F 13 260-279 186 186 186 297 297 297 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 179 179 179 285 285 285 F F F 16 320-339 186 186 186 297 297 297 F F F 17 340-359 180 180 180 286 286 286 F F F 18 360-379 186 186 186 297 297 297 F F F 19 380-399 181 181 181 288 288 288 F F F 20 400-419 186 186 186 297 297 297 F F F 21 420-439 184 184 184 293 293 293 F F F 22 440-459 186 186 186 297 297 297 F F F 23 460-479 176 176 176 282 282 282 F F F 24 480-499 182 182 182 290 290 290 F F F 25 500-519 186 186 186 297 297 297 F F F 26 520-539 186 186 186 297 297 297 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 184 184 184 295 295 295 F F F 29 580-599 182 182 182 289 289 289 F F F Max gen: 186, Max flush: 297 Begin loop 94 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:19.98600835Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.03026704Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.031043938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.031087837Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.03110279Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.031116903Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940} {"msg":"Crucible 76943eb6-d620-4113-b433-c7640c56ab1d has session id: e195ee2a-785e-408f-afd9-57ff5b007003","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.031740898Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Upstairs opts: Upstairs UUID: 76943eb6-d620-4113-b433-c7640c56ab1d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.031777436Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Crucible stats registered with UUID: 76943eb6-d620-4113-b433-c7640c56ab1d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.031802261Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:20.031825426Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032322433Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032345598Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940} {"msg":"76943eb6-d620-4113-b433-c7640c56ab1d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032425806Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032442999Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"0","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032476997Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"0","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.03249192Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"1","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032506263Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"1","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032520126Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"2","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032544071Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"2","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032567736Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"io task","client":"2","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032591071Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"io task","client":"0","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032614647Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"io task","client":"1","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"ds_connection connected from Ok(127.0.0.1:37269)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032637861Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"io task","client":"2","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"ds_connection connected from Ok(127.0.0.1:45791)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032663637Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"io task","client":"1","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"ds_connection connected from Ok(127.0.0.1:34388)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.032745735Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"io task","client":"0","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.033363738Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"0","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.033396065Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"2","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.0339266Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"2","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.034037323Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"0","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.034582601Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.034695396Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.035401918Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"1","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.035893894Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"1","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"[0]R flush_numbers[0..12]: [297, 297, 297, 297, 296, 296, 291, 288, 297, 297, 297, 295]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036413606Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"[0]R generation[0..12]: [186, 186, 186, 186, 185, 185, 183, 181, 186, 186, 186, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036437551Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036452455Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"[1]R flush_numbers[0..12]: [297, 297, 297, 297, 296, 296, 291, 288, 297, 297, 297, 295]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036466307Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"[1]R generation[0..12]: [186, 186, 186, 186, 185, 185, 183, 181, 186, 186, 186, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036479701Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036493324Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"[2]R flush_numbers[0..12]: [297, 297, 297, 297, 296, 296, 291, 288, 297, 297, 297, 295]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036506716Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"[2]R generation[0..12]: [186, 186, 186, 186, 185, 185, 183, 181, 186, 186, 186, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036519939Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036533332Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Max found gen is 187","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036550826Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Generation requested: 188 >= found:187","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036564519Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Next flush: 298","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:33:20.036577322Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036592925Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"":"downstairs","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036606038Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.0366179Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"0","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036630793Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"1","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036643436Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"client":"2","":"downstairs","session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036656088Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036667671Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"76943eb6-d620-4113-b433-c7640c56ab1d is now active with session: e195ee2a-785e-408f-afd9-57ff5b007003","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036679994Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036691336Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940,"session_id":"e195ee2a-785e-408f-afd9-57ff5b007003"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036703049Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940} {"msg":"Activated sub_volume 76943eb6-d620-4113-b433-c7640c56ab1d","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:20.036713881Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 333 len 12288 data: 20 22 24 02/30 Read block 396 len 24576 03/30 Read block 361 len 32768 04/30 Read block 10 len 12288 05/30 Read block 551 len 8192 06/30 Write block 42 len 24576 data: 14 18 20 17 15 15 07/30 Read block 46 len 8192 08/30 Write block 471 len 40960 data: 11 12 13 14 15 11 11 10 10 12 09/30 Write block 527 len 36864 data: 20 16 17 14 17 15 14 16 14 10/30 Read block 343 len 36864 11/30 Write block 11 len 8192 data: 12 13 12/30 Write block 425 len 28672 data: 10 14 16 14 15 17 17 13/30 Read block 37 len 24576 14/30 Read block 490 len 24576 15/30 Write block 345 len 8192 data: 12 7 16/30 Write block 455 len 40960 data: 10 11 13 12 12 14 16 15 16 16 17/30 Read block 113 len 12288 18/30 Read block 592 len 28672 19/30 Read block 62 len 20480 20/30 Read block 495 len 20480 21/30 Read block 581 len 8192 22/30 Read block 230 len 12288 23/30 Read block 487 len 12288 24/30 Write block 184 len 12288 data: 24 20 16 25/30 Read block 441 len 12288 26/30 Write block 584 len 36864 data: 17 18 14 15 13 16 17 18 15 27/30 Read block 513 len 20480 28/30 Read block 456 len 24576 29/30 Write block 167 len 28672 data: 15 16 19 17 15 15 11 30/30 Read block 592 len 16384 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:20.058473406Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1940} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 297 297 297 T T T 1 020-039 186 186 186 297 297 297 F F F 2 040-059 186 186 186 297 297 297 T T T 3 060-079 186 186 186 297 297 297 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 185 185 185 296 296 296 F F F 6 120-139 183 183 183 291 291 291 F F F 7 140-159 181 181 181 288 288 288 F F F 8 160-179 186 186 186 297 297 297 T T T 9 180-199 186 186 186 297 297 297 T T T 10 200-219 186 186 186 297 297 297 F F F 11 220-239 184 184 184 295 295 295 F F F 12 240-259 185 185 185 296 296 296 F F F 13 260-279 186 186 186 297 297 297 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 179 179 179 285 285 285 F F F 16 320-339 186 186 186 297 297 297 T T T 17 340-359 180 180 180 286 286 286 T T T 18 360-379 186 186 186 297 297 297 F F F 19 380-399 181 181 181 288 288 288 F F F 20 400-419 186 186 186 297 297 297 F F F 21 420-439 184 184 184 293 293 293 T T T 22 440-459 186 186 186 297 297 297 T T T 23 460-479 176 176 176 282 282 282 T T T 24 480-499 182 182 182 290 290 290 T T T 25 500-519 186 186 186 297 297 297 F F F 26 520-539 186 186 186 297 297 297 T T T 27 540-559 185 185 185 296 296 296 F F F 28 560-579 184 184 184 295 295 295 F F F 29 580-599 182 182 182 289 289 289 T T T Max gen: 186, Max flush: 297 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 298 298 298 F F F 1 020-039 186 186 186 297 297 297 F F F 2 040-059 187 187 187 298 298 298 F F F 3 060-079 186 186 186 297 297 297 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 185 185 185 296 296 296 F F F 6 120-139 183 183 183 291 291 291 F F F 7 140-159 181 181 181 288 288 288 F F F 8 160-179 187 187 187 298 298 298 F F F 9 180-199 187 187 187 298 298 298 F F F 10 200-219 186 186 186 297 297 297 F F F 11 220-239 184 184 184 295 295 295 F F F 12 240-259 185 185 185 296 296 296 F F F 13 260-279 186 186 186 297 297 297 F F F 14 280-299 186 186 186 297 297 297 F F F 15 300-319 179 179 179 285 285 285 F F F 16 320-339 187 187 187 298 298 298 F F F 17 340-359 187 187 187 298 298 298 F F F 18 360-379 186 186 186 297 297 297 F F F 19 380-399 181 181 181 288 288 288 F F F 20 400-419 186 186 186 297 297 297 F F F 21 420-439 187 187 187 298 298 298 F F F 22 440-459 187 187 187 298 298 298 F F F 23 460-479 187 187 187 298 298 298 F F F 24 480-499 187 187 187 298 298 298 F F F 25 500-519 186 186 186 297 297 297 F F F 26 520-539 187 187 187 298 298 298 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 184 184 184 295 295 295 F F F 29 580-599 187 187 187 298 298 298 F F F Max gen: 187, Max flush: 298 Begin loop 95 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.33664991Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:34.378785582Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.379151641Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.410175775Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.410920986Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.410958984Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.410973767Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.41098833Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948} {"msg":"Crucible 1f0a94d7-69b0-44fb-8094-c644b8483aa9 has session id: 05917b9a-c9ec-420b-9167-809cafd5ece1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.411640221Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Upstairs opts: Upstairs UUID: 1f0a94d7-69b0-44fb-8094-c644b8483aa9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.41168327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Crucible stats registered with UUID: 1f0a94d7-69b0-44fb-8094-c644b8483aa9","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.411706875Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:34.41172733Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412171646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412198832Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948} {"msg":"1f0a94d7-69b0-44fb-8094-c644b8483aa9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412274208Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412292181Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"0","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412320408Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"0","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412334591Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"1","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412348904Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"1","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412366828Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"2","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412382301Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"2","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412395304Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"io task","client":"2","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412408907Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"io task","client":"0","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.41242218Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"io task","client":"1","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"ds_connection connected from Ok(127.0.0.1:33247)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412585115Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"io task","client":"1","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"ds_connection connected from Ok(127.0.0.1:62671)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412623463Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"io task","client":"2","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47844)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.412652279Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"io task","client":"0","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.413251839Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"2","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.413283455Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"1","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.413519866Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"0","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414105203Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"1","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414144541Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"0","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414174858Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"2","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414729678Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414755593Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"[0]R flush_numbers[0..12]: [298, 297, 298, 297, 296, 296, 291, 288, 298, 298, 297, 295]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414776718Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"[0]R generation[0..12]: [187, 186, 187, 186, 185, 185, 183, 181, 187, 187, 186, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414791691Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414805864Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"[1]R flush_numbers[0..12]: [298, 297, 298, 297, 296, 296, 291, 288, 298, 298, 297, 295]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414819487Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"[1]R generation[0..12]: [187, 186, 187, 186, 185, 185, 183, 181, 187, 187, 186, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.41483308Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414855715Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"[2]R flush_numbers[0..12]: [298, 297, 298, 297, 296, 296, 291, 288, 298, 298, 297, 295]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.41488109Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"[2]R generation[0..12]: [187, 186, 187, 186, 185, 185, 183, 181, 187, 187, 186, 184]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414906946Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414924729Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414942153Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Generation requested: 190 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414964778Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.414990134Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.415016339Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"":"downstairs","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.415031713Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.415043955Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"0","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2025-03-17T22:33:34.415075462Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"1","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.415089205Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"client":"2","":"downstairs","session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.415102108Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.415114001Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"1f0a94d7-69b0-44fb-8094-c644b8483aa9 is now active with session: 05917b9a-c9ec-420b-9167-809cafd5ece1","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.415126483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.415138706Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948,"session_id":"05917b9a-c9ec-420b-9167-809cafd5ece1"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.415150698Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948} {"msg":"Activated sub_volume 1f0a94d7-69b0-44fb-8094-c644b8483aa9","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:34.415161581Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 194 len 8192 data: 14 13 02/30 Read block 514 len 40960 03/30 Flush 04/30 Write block 23 len 24576 data: 13 11 7 9 9 12 05/30 Read block 133 len 8192 06/30 Read block 566 len 4096 07/30 Write block 291 len 12288 data: 17 16 12 08/30 Write block 304 len 16384 data: 14 13 14 15 09/30 Flush 10/30 Write block 117 len 16384 data: 16 17 16 14 11/30 Read block 270 len 16384 12/30 Flush 13/30 Write block 153 len 40960 data: 20 23 16 15 14 13 15 15 14 11 14/30 Read block 531 len 24576 15/30 Read block 379 len 24576 16/30 Write block 266 len 20480 data: 16 15 10 10 10 17/30 Read block 192 len 24576 18/30 Flush 19/30 Write block 465 len 12288 data: 14 10 16 20/30 Write block 28 len 36864 data: 13 12 11 11 9 10 11 13 12 21/30 Write block 210 len 28672 data: 17 19 16 10 8 10 12 22/30 Read block 103 len 32768 23/30 Read block 141 len 24576 24/30 Write block 488 len 12288 data: 12 16 17 25/30 Read block 408 len 12288 26/30 Read block 69 len 24576 27/30 Read block 82 len 32768 28/30 Write block 237 len 12288 data: 12 13 11 29/30 Write block 187 len 40960 data: 15 14 12 10 9 10 10 15 14 14 30/30 Read block 19 len 40960 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 3 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:34.435452969Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1948} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 298 298 298 F F F 1 020-039 190 190 190 300 300 300 T T T 2 040-059 187 187 187 298 298 298 F F F 3 060-079 186 186 186 297 297 297 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 190 190 190 301 301 301 F F F 6 120-139 190 190 190 301 301 301 F F F 7 140-159 190 190 190 302 302 302 F F F 8 160-179 190 190 190 302 302 302 F F F 9 180-199 190 190 190 299 299 299 T T T 10 200-219 186 186 186 297 297 297 T T T 11 220-239 184 184 184 295 295 295 T T T 12 240-259 185 185 185 296 296 296 F F F 13 260-279 190 190 190 302 302 302 F F F 14 280-299 190 190 190 300 300 300 F F F 15 300-319 190 190 190 300 300 300 F F F 16 320-339 187 187 187 298 298 298 F F F 17 340-359 187 187 187 298 298 298 F F F 18 360-379 186 186 186 297 297 297 F F F 19 380-399 181 181 181 288 288 288 F F F 20 400-419 186 186 186 297 297 297 F F F 21 420-439 187 187 187 298 298 298 F F F 22 440-459 187 187 187 298 298 298 F F F 23 460-479 187 187 187 298 298 298 T T T 24 480-499 187 187 187 298 298 298 T T T 25 500-519 186 186 186 297 297 297 F F F 26 520-539 187 187 187 298 298 298 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 184 184 184 295 295 295 F F F 29 580-599 187 187 187 298 298 298 F F F Max gen: 190, Max flush: 302 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 298 298 298 F F F 1 020-039 191 191 191 303 303 303 F F F 2 040-059 187 187 187 298 298 298 F F F 3 060-079 186 186 186 297 297 297 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 190 190 190 301 301 301 F F F 6 120-139 190 190 190 301 301 301 F F F 7 140-159 190 190 190 302 302 302 F F F 8 160-179 190 190 190 302 302 302 F F F 9 180-199 191 191 191 303 303 303 F F F 10 200-219 191 191 191 303 303 303 F F F 11 220-239 191 191 191 303 303 303 F F F 12 240-259 185 185 185 296 296 296 F F F 13 260-279 190 190 190 302 302 302 F F F 14 280-299 190 190 190 300 300 300 F F F 15 300-319 190 190 190 300 300 300 F F F 16 320-339 187 187 187 298 298 298 F F F 17 340-359 187 187 187 298 298 298 F F F 18 360-379 186 186 186 297 297 297 F F F 19 380-399 181 181 181 288 288 288 F F F 20 400-419 186 186 186 297 297 297 F F F 21 420-439 187 187 187 298 298 298 F F F 22 440-459 187 187 187 298 298 298 F F F 23 460-479 191 191 191 303 303 303 F F F 24 480-499 191 191 191 303 303 303 F F F 25 500-519 186 186 186 297 297 297 F F F 26 520-539 187 187 187 298 298 298 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 184 184 184 295 295 295 F F F 29 580-599 187 187 187 298 298 298 F F F Max gen: 191, Max flush: 303 Begin loop 96 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.714618756Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.759147424Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.759923071Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.759972852Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.759993477Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.760022713Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956} {"msg":"Crucible 852c61cd-054a-4a28-98af-14cbd84faceb has session id: a8b3c9be-2486-46fd-8f97-c2838d0bea5c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.760645267Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Upstairs opts: Upstairs UUID: 852c61cd-054a-4a28-98af-14cbd84faceb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.760695288Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Crucible stats registered with UUID: 852c61cd-054a-4a28-98af-14cbd84faceb","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.760723834Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:38.760754431Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761259379Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761286576Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956} {"msg":"852c61cd-054a-4a28-98af-14cbd84faceb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761348499Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761366843Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"0","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761395339Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"0","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761409772Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"1","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761423315Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"1","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761524777Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"2","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761563825Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"2","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761592411Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"io task","client":"2","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761660016Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"io task","client":"0","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33609)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761701175Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"io task","client":"2","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761728611Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"io task","client":"1","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48567)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761806088Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"io task","client":"0","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60181)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.761846066Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"io task","client":"1","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.762339633Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"0","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.762381302Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"1","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.763012588Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"1","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.763146327Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"0","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.763183665Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"2","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.763591603Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.763669319Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.763875544Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"2","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"[0]R flush_numbers[0..12]: [298, 303, 298, 297, 296, 301, 301, 302, 302, 303, 303, 303]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764393646Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"[0]R generation[0..12]: [187, 191, 187, 186, 185, 190, 190, 190, 190, 191, 191, 191]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.76441435Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764429254Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"[1]R flush_numbers[0..12]: [298, 303, 298, 297, 296, 301, 301, 302, 302, 303, 303, 303]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764443166Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"[1]R generation[0..12]: [187, 191, 187, 186, 185, 190, 190, 190, 190, 191, 191, 191]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764456379Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764473433Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"[2]R flush_numbers[0..12]: [298, 303, 298, 297, 296, 301, 301, 302, 302, 303, 303, 303]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764487136Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"[2]R generation[0..12]: [187, 191, 187, 186, 185, 190, 190, 190, 190, 191, 191, 191]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764500479Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764513942Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764526504Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764540467Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.76455342Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764566043Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"":"downstairs","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764579136Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764590938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"0","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764603851Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"client":"1","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764616494Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"clientWait for a query_work_queue command to finish before sending IO ":"2","":"downstairs","session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764637999Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764649921Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"852c61cd-054a-4a28-98af-14cbd84faceb is now active with session: a8b3c9be-2486-46fd-8f97-c2838d0bea5c","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764662614Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764677687Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956,"session_id":"a8b3c9be-2486-46fd-8f97-c2838d0bea5c"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.76469014Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956} {"msg":"Activated sub_volume 852c61cd-054a-4a28-98af-14cbd84faceb","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:38.764700922Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 57 len 16384 data: 14 15 14 14 02/30 Read block 395 len 32768 03/30 Read block 137 len 8192 04/30 Flush 05/30 Write block 312 len 24576 data: 11 14 13 13 15 15 06/30 Flush 07/30 Write block 231 len 32768 data: 15 15 17 15 16 14 13 14 08/30 Read block 93 len 36864 09/30 Flush 10/30 Write block 27 len 4096 data: 10 11/30 Write block 30 len 32768 data: 12 12 10 11 12 14 13 13 12/30 Write block 570 len 4096 data: 16 13/30 Write block 257 len 36864 data: 14 17 16 15 11 10 14 15 18 14/30 Read block 137 len 16384 15/30 Flush 16/30 Write block 442 len 32768 data: 17 16 15 16 17 15 13 13 17/30 Write block 494 len 4096 data: 13 18/30 Flush 19/30 Flush 20/30 Write block 358 len 20480 data: 19 19 20 13 15 21/30 Read block 503 len 24576 22/30 Read block 466 len 32768 23/30 Write block 171 len 4096 data: 16 24/30 Write block 459 len 4096 data: 13 25/30 Write block 518 len 12288 data: 16 13 10 26/30 Write block 329 len 40960 data: 11 12 17 19 21 23 25 19 16 14 27/30 Write block 313 len 32768 data: 15 14 14 16 16 10 15 16 28/30 Read block 23 len 32768 29/30 Read block 42 len 28672 30/30 Read block 590 len 28672 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 5 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 11 10 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:38.784011579Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1956} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 298 298 298 F F F 1 020-039 192 192 192 307 307 307 F F F 2 040-059 192 192 192 304 304 304 F F F 3 060-079 192 192 192 304 304 304 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 190 190 190 301 301 301 F F F 6 120-139 190 190 190 301 301 301 F F F 7 140-159 190 190 190 302 302 302 F F F 8 160-179 190 190 190 302 302 302 T T T 9 180-199 191 191 191 303 303 303 F F F 10 200-219 191 191 191 303 303 303 F F F 11 220-239 192 192 192 306 306 306 F F F 12 240-259 192 192 192 307 307 307 F F F 13 260-279 192 192 192 307 307 307 F F F 14 280-299 190 190 190 300 300 300 F F F 15 300-319 192 192 192 305 305 305 T T T 16 320-339 187 187 187 298 298 298 T T T 17 340-359 187 187 187 298 298 298 T T T 18 360-379 186 186 186 297 297 297 T T T 19 380-399 181 181 181 288 288 288 F F F 20 400-419 186 186 186 297 297 297 F F F 21 420-439 187 187 187 298 298 298 F F F 22 440-459 192 192 192 308 308 308 T T T 23 460-479 191 191 191 303 303 303 F F F 24 480-499 192 192 192 308 308 308 F F F 25 500-519 186 186 186 297 297 297 T T T 26 520-539 187 187 187 298 298 298 T T T 27 540-559 185 185 185 296 296 296 F F F 28 560-579 192 192 192 307 307 307 F F F 29 580-599 187 187 187 298 298 298 F F F Max gen: 192, Max flush: 308 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 298 298 298 F F F 1 020-039 192 192 192 307 307 307 F F F 2 040-059 192 192 192 304 304 304 F F F 3 060-079 192 192 192 304 304 304 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 190 190 190 301 301 301 F F F 6 120-139 190 190 190 301 301 301 F F F 7 140-159 190 190 190 302 302 302 F F F 8 160-179 193 193 193 309 309 309 F F F 9 180-199 191 191 191 303 303 303 F F F 10 200-219 191 191 191 303 303 303 F F F 11 220-239 192 192 192 306 306 306 F F F 12 240-259 192 192 192 307 307 307 F F F 13 260-279 192 192 192 307 307 307 F F F 14 280-299 190 190 190 300 300 300 F F F 15 300-319 193 193 193 309 309 309 F F F 16 320-339 193 193 193 309 309 309 F F F 17 340-359 193 193 193 309 309 309 F F F 18 360-379 193 193 193 309 309 309 F F F 19 380-399 181 181 181 288 288 288 F F F 20 400-419 186 186 186 297 297 297 F F F 21 420-439 187 187 187 298 298 298 F F F 22 440-459 193 193 193 309 309 309 F F F 23 460-479 191 191 191 303 303 303 F F F 24 480-499 192 192 192 308 308 308 F F F 25 500-519 193 193 193 309 309 309 F F F 26 520-539 193 193 193 309 309 309 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 192 192 192 307 307 307 F F F 29 580-599 187 187 187 298 298 298 F F F Max gen: 193, Max flush: 309 Begin loop 97 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.042093137Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.085705888Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.086314129Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.086357599Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.086376312Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.086391096Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964} {"msg":"Crucible 42d1d6ef-3ea4-41ed-8b95-c6ba74d40651 has session id: 81c5fb59-bf5f-4923-898b-8e1078591d7f","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.086938284Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"Upstairs opts: Upstairs UUID: 42d1d6ef-3ea4-41ed-8b95-c6ba74d40651, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.086993566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"Crucible stats registered with UUID: 42d1d6ef-3ea4-41ed-8b95-c6ba74d40651","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087023362Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:53.087049678Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087363565Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087387651Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964} {"msg":"42d1d6ef-3ea4-41ed-8b95-c6ba74d40651 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087445893Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087471839Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"0","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087500855Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"0","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087515298Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"1","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087529571Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"1","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087543364Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"2","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.08756784Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"2","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087588874Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087612939Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"io task","client":"0","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087636704Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"io task","client":"1","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:53.08766408Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:53.087691786Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:33:53.087719202Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64478)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087743367Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"io task","client":"0","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58949)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087818934Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"io task","client":"1","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.087879697Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.088300128Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"0","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.088328784Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"1","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.088932584Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"1","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.089060802Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"0","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.08951362Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:33:53.0896079Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.088931384Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46287)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.089056611Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"io task","client":"2","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.090161259Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"2","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.09072351Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"2","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"[0]R flush_numbers[0..12]: [298, 307, 304, 304, 296, 301, 301, 302, 309, 303, 303, 306]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091405328Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"[0]R generation[0..12]: [187, 192, 192, 192, 185, 190, 190, 190, 193, 191, 191, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091434984Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091454228Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"[1]R flush_numbers[0..12]: [298, 307, 304, 304, 296, 301, 301, 302, 309, 303, 303, 306]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091471432Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"[1]R generation[0..12]: [187, 192, 192, 192, 185, 190, 190, 190, 193, 191, 191, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091488286Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091505489Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"[2]R flush_numbers[0..12]: [298, 307, 304, 304, 296, 301, 301, 302, 309, 303, 303, 306]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091522303Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"[2]R generation[0..12]: [187, 192, 192, 192, 185, 190, 190, 190, 193, 191, 191, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091539117Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091563412Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091580325Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091596439Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091612372Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091628306Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"":"downstairs","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091659012Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091675106Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"0","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091691679Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"1","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":Wait for a query_work_queue command to finish before sending IO "Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091716445Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"client":"2","":"downstairs","session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091732628Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091747382Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"42d1d6ef-3ea4-41ed-8b95-c6ba74d40651 is now active with session: 81c5fb59-bf5f-4923-898b-8e1078591d7f","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091762715Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091777108Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964,"session_id":"81c5fb59-bf5f-4923-898b-8e1078591d7f"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091791691Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964} {"msg":"Activated sub_volume 42d1d6ef-3ea4-41ed-8b95-c6ba74d40651","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:03.091805154Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 401 len 40960 data: 11 13 15 15 17 18 19 18 16 16 02/30 Read block 439 len 28672 03/30 Write block 147 len 40960 data: 15 17 19 18 21 20 21 24 17 16 04/30 Write block 340 len 16384 data: 13 12 15 14 05/30 Write block 359 len 40960 data: 20 21 14 16 17 14 16 11 10 10 06/30 Read block 72 len 12288 07/30 Write block 258 len 4096 data: 18 08/30 Read block 254 len 12288 09/30 Read block 195 len 8192 10/30 Flush 11/30 Write block 467 len 8192 data: 17 16 12/30 Read block 0 len 16384 13/30 Read block 133 len 28672 14/30 Read block 332 len 8192 15/30 Write block 317 len 40960 data: 17 11 16 17 16 15 15 15 16 15 16/30 Write block 380 len 40960 data: 14 14 13 13 12 14 15 13 15 15 17/30 Write block 441 len 40960 data: 13 18 17 16 17 18 16 14 14 10 18/30 Read block 21 len 24576 19/30 Read block 576 len 20480 20/30 Read block 288 len 16384 21/30 Write block 47 len 40960 data: 16 19 19 17 14 15 13 13 15 13 22/30 Write block 498 len 32768 data: 11 14 14 19 18 18 16 15 23/30 Read block 448 len 4096 24/30 Read block 450 len 12288 25/30 Write block 298 len 28672 data: 15 12 14 14 16 16 15 26/30 Flush 27/30 Write block 100 len 24576 data: 11 11 12 10 11 9 28/30 Write block 338 len 16384 data: 15 14 14 13 29/30 Write block 184 len 28672 data: 25 21 17 16 15 13 11 30/30 Read block 499 len 20480 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 2 4 4 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:34:03.113235048Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1964} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 298 298 298 F F F 1 020-039 192 192 192 307 307 307 F F F 2 040-059 194 194 194 311 311 311 F F F 3 060-079 192 192 192 304 304 304 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 190 190 190 301 301 301 T T T 6 120-139 190 190 190 301 301 301 F F F 7 140-159 194 194 194 310 310 310 F F F 8 160-179 193 193 193 309 309 309 F F F 9 180-199 191 191 191 303 303 303 T T T 10 200-219 191 191 191 303 303 303 F F F 11 220-239 192 192 192 306 306 306 F F F 12 240-259 194 194 194 310 310 310 F F F 13 260-279 192 192 192 307 307 307 F F F 14 280-299 194 194 194 311 311 311 F F F 15 300-319 194 194 194 311 311 311 F F F 16 320-339 194 194 194 311 311 311 T T T 17 340-359 194 194 194 310 310 310 T T T 18 360-379 194 194 194 310 310 310 F F F 19 380-399 194 194 194 311 311 311 F F F 20 400-419 194 194 194 310 310 310 F F F 21 420-439 187 187 187 298 298 298 F F F 22 440-459 194 194 194 311 311 311 F F F 23 460-479 194 194 194 311 311 311 F F F 24 480-499 194 194 194 311 311 311 F F F 25 500-519 194 194 194 311 311 311 F F F 26 520-539 193 193 193 309 309 309 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 192 192 192 307 307 307 F F F 29 580-599 187 187 187 298 298 298 F F F Max gen: 194, Max flush: 311 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 298 298 298 F F F 1 020-039 192 192 192 307 307 307 F F F 2 040-059 194 194 194 311 311 311 F F F 3 060-079 192 192 192 304 304 304 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 195 195 195 312 312 312 F F F 6 120-139 190 190 190 301 301 301 F F F 7 140-159 194 194 194 310 310 310 F F F 8 160-179 193 193 193 309 309 309 F F F 9 180-199 195 195 195 312 312 312 F F F 10 200-219 191 191 191 303 303 303 F F F 11 220-239 192 192 192 306 306 306 F F F 12 240-259 194 194 194 310 310 310 F F F 13 260-279 192 192 192 307 307 307 F F F 14 280-299 194 194 194 311 311 311 F F F 15 300-319 194 194 194 311 311 311 F F F 16 320-339 195 195 195 312 312 312 F F F 17 340-359 195 195 195 312 312 312 F F F 18 360-379 194 194 194 310 310 310 F F F 19 380-399 194 194 194 311 311 311 F F F 20 400-419 194 194 194 310 310 310 F F F 21 420-439 187 187 187 298 298 298 F F F 22 440-459 194 194 194 311 311 311 F F F 23 460-479 194 194 194 311 311 311 F F F 24 480-499 194 194 194 311 311 311 F F F 25 500-519 194 194 194 311 311 311 F F F 26 520-539 193 193 193 309 309 309 F F F 27 540-559 185 185 185 296 296 296 F F F 28 560-579 192 192 192 307 307 307 F F F 29 580-599 187 187 187 298 298 298 F F F Max gen: 195, Max flush: 312 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.35220463Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:34:07.394292632Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.394713033Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.42598552Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.42663335Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.426667357Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.42668123Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.426701275Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972} {"msg":"Crucible 2c9f6fcf-6fba-45fb-b0e9-58850b7c97ea has session id: f48ce6e9-b6c1-455f-86e1-41992c944ce6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.427305775Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Upstairs opts: Upstairs UUID: 2c9f6fcf-6fba-45fb-b0e9-58850b7c97ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.427343483Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Crucible stats registered with UUID: 2c9f6fcf-6fba-45fb-b0e9-58850b7c97ea","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.427359867Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:34:07.427372649Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.427880679Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.427900883Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972} {"msg":"2c9f6fcf-6fba-45fb-b0e9-58850b7c97ea active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.427971299Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.427989033Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"0","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.428030342Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"0","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.428047345Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"1","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.428061458Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"1","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.428076821Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"2","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.428100297Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"2","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.428123422Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"io task","client":"2","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.428153588Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"io task","client":"0","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.428177903Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"io task","client":"1","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42327)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.428203629Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"io task","client":"2","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"ds_connection connected from Ok(127.0.0.1:43831)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.428229814Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"io task","client":"0","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"ds_connection connected from Ok(127.0.0.1:51492)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.42825431Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"io task","client":"1","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.429025426Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"1","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.429057013Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"2","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.429083429Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"0","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.429625826Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"1","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.429653912Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"0","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.429677087Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"2","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430219884Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430333919Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"[0]R flush_numbers[0..12]: [298, 307, 311, 304, 296, 312, 301, 310, 309, 312, 303, 306]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430395462Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"[0]R generation[0..12]: [187, 192, 194, 192, 185, 195, 190, 194, 193, 195, 191, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430426908Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430442522Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"[1]R flush_numbers[0..12]: [298, 307, 311, 304, 296, 312, 301, 310, 309, 312, 303, 306]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430456495Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"[1]R generation[0..12]: [187, 192, 194, 192, 185, 195, 190, 194, 193, 195, 191, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430469938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430484571Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"[2]R flush_numbers[0..12]: [298, 307, 311, 304, 296, 312, 301, 310, 309, 312, 303, 306]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430498334Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"[2]R generation[0..12]: [187, 192, 194, 192, 185, 195, 190, 194, 193, 195, 191, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430511657Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.4305252Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430537892Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430550745Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430564048Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:34:07.430576791Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"":"downstairs","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430596665Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430608408Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"0","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430623491Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"1","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430636854Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"client":"2","":"downstairs","session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430649987Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430661809Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"2c9f6fcf-6fba-45fb-b0e9-58850b7c97ea is now active with session: f48ce6e9-b6c1-455f-86e1-41992c944ce6","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430674122Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430685624Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972,"session_id":"f48ce6e9-b6c1-455f-86e1-41992c944ce6"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.430697527Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972} {"msg":"Activated sub_volume 2c9f6fcf-6fba-45fb-b0e9-58850b7c97ea","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:07.43070938Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 178 len 4096 data: 15 02/30 Read block 313 len 12288 03/30 Write block 205 len 32768 data: 15 15 13 14 17 18 20 17 04/30 Read block 335 len 20480 05/30 Read block 261 len 28672 06/30 Read block 345 len 8192 07/30 Write block 112 len 12288 data: 12 14 16 08/30 Write block 557 len 32768 data: 16 18 19 21 22 22 22 21 09/30 Read block 519 len 16384 10/30 Read block 468 len 32768 11/30 Read block 303 len 36864 12/30 Read block 242 len 40960 13/30 Read block 122 len 8192 14/30 Write block 127 len 36864 data: 9 11 13 12 15 14 14 13 14 15/30 Write block 404 len 28672 data: 16 18 19 20 19 17 17 16/30 Write block 185 len 20480 data: 22 18 17 16 14 17/30 Read block 287 len 20480 18/30 Flush 19/30 Write block 490 len 12288 data: 18 16 17 20/30 Read block 274 len 16384 21/30 Flush 22/30 Write block 245 len 32768 data: 11 15 16 14 14 14 15 16 23/30 Write block 378 len 8192 data: 14 17 24/30 Read block 463 len 40960 25/30 Read block 158 len 36864 26/30 Write block 196 len 28672 data: 15 16 16 14 17 19 16 27/30 Read block 274 len 8192 28/30 Read block 574 len 8192 29/30 Write block 308 len 36864 data: 17 15 11 13 12 16 15 15 17 30/30 Read block 189 len 40960 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 9 8 7 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:34:07.451866965Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1972} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 298 298 298 F F F 1 020-039 192 192 192 307 307 307 F F F 2 040-059 194 194 194 311 311 311 F F F 3 060-079 192 192 192 304 304 304 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 196 196 196 313 313 313 F F F 6 120-139 196 196 196 313 313 313 F F F 7 140-159 194 194 194 310 310 310 F F F 8 160-179 196 196 196 313 313 313 F F F 9 180-199 196 196 196 313 313 313 T T T 10 200-219 196 196 196 313 313 313 T T T 11 220-239 192 192 192 306 306 306 F F F 12 240-259 194 194 194 310 310 310 T T T 13 260-279 192 192 192 307 307 307 F F F 14 280-299 194 194 194 311 311 311 F F F 15 300-319 194 194 194 311 311 311 T T T 16 320-339 195 195 195 312 312 312 F F F 17 340-359 195 195 195 312 312 312 F F F 18 360-379 194 194 194 310 310 310 T T T 19 380-399 194 194 194 311 311 311 F F F 20 400-419 196 196 196 313 313 313 F F F 21 420-439 187 187 187 298 298 298 F F F 22 440-459 194 194 194 311 311 311 F F F 23 460-479 194 194 194 311 311 311 F F F 24 480-499 196 196 196 314 314 314 F F F 25 500-519 194 194 194 311 311 311 F F F 26 520-539 193 193 193 309 309 309 F F F 27 540-559 196 196 196 313 313 313 F F F 28 560-579 196 196 196 313 313 313 F F F 29 580-599 187 187 187 298 298 298 F F F Max gen: 196, Max flush: 314 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 298 298 298 F F F 1 020-039 192 192 192 307 307 307 F F F 2 040-059 194 194 194 311 311 311 F F F 3 060-079 192 192 192 304 304 304 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 196 196 196 313 313 313 F F F 6 120-139 196 196 196 313 313 313 F F F 7 140-159 194 194 194 310 310 310 F F F 8 160-179 196 196 196 313 313 313 F F F 9 180-199 197 197 197 315 315 315 F F F 10 200-219 197 197 197 315 315 315 F F F 11 220-239 192 192 192 306 306 306 F F F 12 240-259 197 197 197 315 315 315 F F F 13 260-279 192 192 192 307 307 307 F F F 14 280-299 194 194 194 311 311 311 F F F 15 300-319 197 197 197 315 315 315 F F F 16 320-339 195 195 195 312 312 312 F F F 17 340-359 195 195 195 312 312 312 F F F 18 360-379 197 197 197 315 315 315 F F F 19 380-399 194 194 194 311 311 311 F F F 20 400-419 196 196 196 313 313 313 F F F 21 420-439 187 187 187 298 298 298 F F F 22 440-459 194 194 194 311 311 311 F F F 23 460-479 194 194 194 311 311 311 F F F 24 480-499 196 196 196 314 314 314 F F F 25 500-519 194 194 194 311 311 311 F F F 26 520-539 193 193 193 309 309 309 F F F 27 540-559 196 196 196 313 313 313 F F F 28 560-579 196 196 196 313 313 313 F F F 29 580-599 187 187 187 298 298 298 F F F Max gen: 197, Max flush: 315 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.723114602Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.767982503Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.768768673Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"acc6b11fa639390710fd9a0449e05c901b048ed2\",\n git_commit_timestamp: \"2025-03-17T22:02:02.000000000Z\",\n git_branch: \"renovate/itertools-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.768810722Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.768833907Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.768851891Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980} {"msg":"Crucible a3e7d03a-bbb2-46b6-bb40-5d9cbaae8840 has session id: 24521c9a-ec0b-4419-8cf2-e1e6b64788af","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.76954031Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Upstairs opts: Upstairs UUID: a3e7d03a-bbb2-46b6-bb40-5d9cbaae8840, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.76958744Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Crucible stats registered with UUID: a3e7d03a-bbb2-46b6-bb40-5d9cbaae8840","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.769612685Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:34:11.76963355Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.770095919Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.770121125Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980} {"msg":"a3e7d03a-bbb2-46b6-bb40-5d9cbaae8840 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.770184389Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.770202092Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"0","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.77023564Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"0","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.770255024Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"1","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.770343583Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"1","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.770379751Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"2","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.770437463Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"2","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.77046693Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"io task","client":"2","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.770491135Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"io task","client":"0","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.77051655Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"io task","client":"1","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"ds_connection connected from Ok(127.0.0.1:55288)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.770543426Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"io task","client":"2","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"ds_connection connected from Ok(127.0.0.1:34660)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.770569052Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"io task","client":"0","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"ds_connection connected from Ok(127.0.0.1:45824)","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.770593237Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"io task","client":"1","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.771192076Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"1","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.771220712Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"0","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.771888436Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"1","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.771933336Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"2","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.772008872Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"0","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.772567473Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.772670675Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.772697151Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"2","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"[0]R flush_numbers[0..12]: [298, 307, 311, 304, 296, 313, 313, 310, 313, 315, 315, 306]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.77325262Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"[0]R generation[0..12]: [187, 192, 194, 192, 185, 196, 196, 194, 196, 197, 197, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773274245Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773289128Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"[1]R flush_numbers[0..12]: [298, 307, 311, 304, 296, 313, 313, 310, 313, 315, 315, 306]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773302801Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"[1]R generation[0..12]: [187, 192, 194, 192, 185, 196, 196, 194, 196, 197, 197, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773316504Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773335678Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"[2]R flush_numbers[0..12]: [298, 307, 311, 304, 296, 313, 313, 310, 313, 315, 315, 306]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773352642Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"[2]R generation[0..12]: [187, 192, 194, 192, 185, 196, 196, 194, 196, 197, 197, 192]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773369296Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773386009Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773407214Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773423988Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-17T22:34:11.773439631Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773460845Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"":"downstairs","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773476729Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773490842Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"0","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773506905Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"1","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773522349Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"client":"2","":"downstairs","session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773538222Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773553015Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"a3e7d03a-bbb2-46b6-bb40-5d9cbaae8840 is now active with session: 24521c9a-ec0b-4419-8cf2-e1e6b64788af","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773568389Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773592364Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980,"session_id":"24521c9a-ec0b-4419-8cf2-e1e6b64788af"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-03-17T22:34:11.773610148Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980} {"msg":"Activated sub_volume a3e7d03a-bbb2-46b6-bb40-5d9cbaae8840","v":0,"name":"crucible","level":30,"time":"2025-03-17T22:34:11.773627812Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 516 len 24576 data: 11 17 17 14 11 12 02/30 Read block 31 len 40960 03/30 Flush 04/30 Write block 109 len 16384 data: 10 10 10 13 05/30 Read block 235 len 36864 06/30 Read block 360 len 40960 07/30 Read block 525 len 4096 08/30 Read block 495 len 16384 09/30 Write block 236 len 12288 data: 15 14 15 10/30 Write block 372 len 36864 data: 9 10 10 13 11 13 15 18 15 11/30 Read block 224 len 12288 12/30 Read block 232 len 24576 13/30 Write block 577 len 24576 data: 17 18 19 14 15 15 14/30 Read block 252 len 24576 15/30 Read block 567 len 32768 16/30 Write block 187 len 12288 data: 18 17 15 17/30 Flush 18/30 Write block 232 len 36864 data: 16 18 16 17 16 15 16 12 12 19/30 Read block 362 len 16384 20/30 Read block 546 len 28672 21/30 Read block 208 len 12288 22/30 Read block 501 len 20480 23/30 Write block 208 len 8192 data: 15 18 24/30 Read block 251 len 20480 25/30 Read block 203 len 16384 26/30 Read block 379 len 28672 27/30 Read block 70 len 20480 28/30 Write block 441 len 20480 data: 14 19 18 17 18 29/30 Write block 431 len 32768 data: 18 16 16 15 15 13 11 10 30/30 Write block 22 len 4096 data: 12 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 9 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-17T22:34:11.79429566Z","hostname":"w-01JPK2XVX1460JZZ10JG7J7E5Z","pid":1980} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 298 298 298 F F F 1 020-039 192 192 192 307 307 307 T T T 2 040-059 194 194 194 311 311 311 F F F 3 060-079 192 192 192 304 304 304 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 198 198 198 317 317 317 F F F 6 120-139 196 196 196 313 313 313 F F F 7 140-159 194 194 194 310 310 310 F F F 8 160-179 196 196 196 313 313 313 F F F 9 180-199 198 198 198 317 317 317 F F F 10 200-219 197 197 197 315 315 315 T T T 11 220-239 198 198 198 317 317 317 T T T 12 240-259 197 197 197 315 315 315 T T T 13 260-279 192 192 192 307 307 307 F F F 14 280-299 194 194 194 311 311 311 F F F 15 300-319 197 197 197 315 315 315 F F F 16 320-339 195 195 195 312 312 312 F F F 17 340-359 195 195 195 312 312 312 F F F 18 360-379 198 198 198 317 317 317 F F F 19 380-399 198 198 198 317 317 317 F F F 20 400-419 196 196 196 313 313 313 F F F 21 420-439 187 187 187 298 298 298 T T T 22 440-459 194 194 194 311 311 311 T T T 23 460-479 194 194 194 311 311 311 F F F 24 480-499 196 196 196 314 314 314 F F F 25 500-519 198 198 198 316 316 316 F F F 26 520-539 198 198 198 316 316 316 F F F 27 540-559 196 196 196 313 313 313 F F F 28 560-579 198 198 198 317 317 317 F F F 29 580-599 198 198 198 317 317 317 F F F Max gen: 198, Max flush: 317 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 298 298 298 F F F 1 020-039 199 199 199 318 318 318 F F F 2 040-059 194 194 194 311 311 311 F F F 3 060-079 192 192 192 304 304 304 F F F 4 080-099 185 185 185 296 296 296 F F F 5 100-119 198 198 198 317 317 317 F F F 6 120-139 196 196 196 313 313 313 F F F 7 140-159 194 194 194 310 310 310 F F F 8 160-179 196 196 196 313 313 313 F F F 9 180-199 198 198 198 317 317 317 F F F 10 200-219 199 199 199 318 318 318 F F F 11 220-239 199 199 199 318 318 318 F F F 12 240-259 199 199 199 318 318 318 F F F 13 260-279 192 192 192 307 307 307 F F F 14 280-299 194 194 194 311 311 311 F F F 15 300-319 197 197 197 315 315 315 F F F 16 320-339 195 195 195 312 312 312 F F F 17 340-359 195 195 195 312 312 312 F F F 18 360-379 198 198 198 317 317 317 F F F 19 380-399 198 198 198 317 317 317 F F F 20 400-419 196 196 196 313 313 313 F F F 21 420-439 199 199 199 318 318 318 F F F 22 440-459 199 199 199 318 318 318 F F F 23 460-479 194 194 194 311 311 311 F F F 24 480-499 196 196 196 314 314 314 F F F 25 500-519 198 198 198 316 316 316 F F F 26 520-539 198 198 198 316 316 316 F F F 27 540-559 196 196 196 313 313 313 F F F 28 560-579 198 198 198 317 317 317 F F F 29 580-599 198 198 198 317 317 317 F F F Max gen: 199, Max flush: 318 15:35 Test duration Test completed real 15:34.694267811 user 50.372737572 sys 19.771000481 trap 0.066881738 tflt 0.092229476 dflt 0.244354994 kflt 0.005166017 lock 24:46:40.655416308 slp 1:16:17.365162013 lat 5.873324266 stop 0.063721785